def create(self, email, password, role="<STR_LIT:user>", public=True, **kwargs):

kwargs["<STR_LIT:email>"] = email<EOL>kwargs["<STR_LIT:password>"] = password<EOL>kwargs["<STR_LIT>"] = role<EOL>kwargs["<STR_LIT>"] = public<EOL>self.metadata = self.db.create(<EOL>self.path, kwargs).json()<EOL>

Creates the given user - using the passed in email and password.

        You can also set other default properties by passing in the relevant information::

            usr.create("my@email","mypass",description="I like trains.")

        Furthermore, ConnectorDB permits immediate initialization of an entire user tree,
        so that you can create all relevant devices and streams in one go::

            usr.create("my@email","mypass",devices={
                "device1": {
                    "nickname": "My train",
                    "streams": {
                        "stream1": {
                            "schema": "{\"type\":\"string\"}",
                            "datatype": "train.choochoo"
                        }
                    },
                }
            })

        The user and meta devices are created by default. If you want to add streams to the user device,
        use the "streams" option in place of devices in create.


self.set({"<STR_LIT:password>": new_password})<EOL>


result = self.db.read(self.path, {"<STR_LIT:q>": "<STR_LIT>"})<EOL>if result is None or result.json() is None:<EOL><INDENT>return []<EOL><DEDENT>devices = []<EOL>for d in result.json():<EOL><INDENT>dev = self[d["<STR_LIT:name>"]]<EOL>dev.metadata = d<EOL>devices.append(dev)<EOL><DEDENT>return devices<EOL>

Returns the list of devices that belong to the user


def streams(self, public=False, downlink=False, visible=True):

result = self.db.read(self.path, {"<STR_LIT:q>": "<STR_LIT>",<EOL>"<STR_LIT>": str(public).lower(),<EOL>"<STR_LIT>": str(downlink).lower(),<EOL>"<STR_LIT>": str(visible).lower()})<EOL>if result is None or result.json() is None:<EOL><INDENT>return []<EOL><DEDENT>streams = []<EOL>for d in result.json():<EOL><INDENT>s = self[d["<STR_LIT>"]][d["<STR_LIT:name>"]]<EOL>s.metadata = d<EOL>streams.append(s)<EOL><DEDENT>return streams<EOL>

Returns the list of streams that belong to the user.
        The list can optionally be filtered in 3 ways:
            - public: when True, returns only streams belonging to public devices
            - downlink: If True, returns only downlink streams
            - visible: If True (default), returns only streams of visible devices


return Device(self.db, self.path + "<STR_LIT:/>" + device_name)<EOL>


return "<STR_LIT>" % (self.path, )<EOL>

Returns a string representation of the user


exportInfoFile = os.path.join(directory, "<STR_LIT>")<EOL>if os.path.exists(directory):<EOL><INDENT>if not os.path.exists(exportInfoFile):<EOL><INDENT>raise FileExistsError(<EOL>"<STR_LIT>")<EOL><DEDENT>with open(exportInfoFile) as f:<EOL><INDENT>exportInfo = json.load(f)<EOL><DEDENT>if exportInfo["<STR_LIT>"] != <NUM_LIT:1>:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>os.mkdir(directory)<EOL>with open(exportInfoFile, "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(<EOL>{"<STR_LIT>": <NUM_LIT:1>, "<STR_LIT>": self.db.get("<STR_LIT>").text}, f)<EOL><DEDENT><DEDENT>udir = os.path.join(directory, self.name)<EOL>os.mkdir(udir)<EOL>with open(os.path.join(udir, "<STR_LIT>"), "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(self.data, f)<EOL><DEDENT>for d in self.devices():<EOL><INDENT>d.export(os.path.join(udir, d.name))<EOL><DEDENT>

Exports the ConnectorDB user into the given directory.
        The resulting export can be imported by using the import command(cdb.import(directory)),

        Note that Python cannot export passwords, since the REST API does
        not expose password hashes. Therefore, the imported user will have
        password same as username.

        The user export function is different than device and stream exports because
        it outputs a format compatible directly with connectorDB's import functionality:

            connectordb import < mydatabase > <directory >

        This also means that you can export multiple users into the same directory without issue


<EOL>with open(os.path.join(directory, "<STR_LIT>"), "<STR_LIT:r>") as f:<EOL><INDENT>ddata = json.load(f)<EOL><DEDENT>d = self[ddata["<STR_LIT:name>"]]<EOL>dname = ddata["<STR_LIT:name>"]<EOL>del ddata["<STR_LIT:name>"]<EOL>if dname == "<STR_LIT>":<EOL><INDENT>return<EOL><DEDENT>elif dname == "<STR_LIT:user>":<EOL><INDENT>d.set(ddata)<EOL><DEDENT>elif d.exists():<EOL><INDENT>raise ValueError("<STR_LIT>" + d.name + "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>d.create(**ddata)<EOL><DEDENT>for name in os.listdir(directory):<EOL><INDENT>sdir = os.path.join(directory, name)<EOL>if os.path.isdir(sdir):<EOL><INDENT>d.import_stream(sdir)<EOL><DEDENT><DEDENT>

Imports a device from the given directory. You export the device
        by using device.export()

        There are two special cases: user and meta devices.
        If the device name is meta, import_device will not do anything.
        If the device name is "user", import_device will overwrite the user device
        even if it exists already.


if "<STR_LIT:email>" in self.data:<EOL><INDENT>return self.data["<STR_LIT:email>"]<EOL><DEDENT>return None<EOL>


self.set({"<STR_LIT:email>": new_email})<EOL>


if "<STR_LIT>" in self.data:<EOL><INDENT>return self.data["<STR_LIT>"]<EOL><DEDENT>return None<EOL>

gets whether the user is public
        (this means different things based on connectordb permissions setup - connectordb.com
        has this be whether the user is publically visible. Devices are individually public / private.)


self.set({"<STR_LIT>": new_public})<EOL>

Attempts to set whether the user is public


Gets the role of the user. This is the permissions level that the user has. It might
        not be accessible depending on the permissions setup of ConnectorDB. Returns None if not accessible


self.set({"<STR_LIT>": new_role})<EOL>


list.__init__(self, data)<EOL>


def sort(self, f=lambda d: d["<STR_LIT:t>"]):

list.sort(self, key=f)<EOL>return self<EOL>

Sort here works by sorting by timestamp by default


return list(map(lambda x: x["<STR_LIT:d>"], self.raw()))<EOL>

Returns just the data portion of the datapoints as a list


return list(map(lambda x: datetime.datetime.fromtimestamp(x["<STR_LIT:t>"]), self.raw()))<EOL>

Returns just the timestamp portion of the datapoints as a list.
        The timestamps are in python datetime's date format.


self.extend(array)<EOL>self.sort()<EOL>return self<EOL>

Adds the given array of datapoints to the generator.
        It assumes that the datapoints are formatted correctly for ConnectorDB, meaning
        that they are in the format::

            [{"t": unix timestamp, "d": data}]

        The data does NOT need to be sorted by timestamp - this function sorts it for you


return list.__getitem__(self, slice(None, None))<EOL>

Returns array as a raw python array. For cases where for some reason
        the DatapointArray wrapper does not work for you


with open(filename, "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(self, f)<EOL><DEDENT>

Writes the data to the given file::

            DatapointArray([{"t": unix timestamp, "d": data}]).writeJSON("myfile.json")

        The data can later be loaded using loadJSON.


with open(filename, "<STR_LIT:r>") as f:<EOL><INDENT>self.merge(json.load(f))<EOL><DEDENT>return self<EOL>

Adds the data from a JSON file. The file is expected to be in datapoint format::

            d = DatapointArray().loadJSON("myfile.json")


self.loadJSON(os.path.join(folder, "<STR_LIT>"))<EOL>return self<EOL>

Adds the data from a ConnectorDB export. If it is a stream export, then the folder
        is the location of the export. If it is a device export, then the folder is the export folder
        with the stream name as a subdirectory

        If it is a user export, you will use the path of the export folder, with the user/device/stream
        appended to the end::

            myuser.export("./exportdir")
            DatapointArray().loadExport("./exportdir/username/devicename/streamname")


raw = self.raw()<EOL>for i in range(len(raw)):<EOL><INDENT>raw[i]["<STR_LIT:t>"] += t<EOL><DEDENT>return self<EOL>

Shifts all timestamps in the datapoint array by the given number of seconds.
        It is the same as the 'tshift' pipescript transform.

        Warning: The shift is performed in-place! This means that it modifies the underlying array::

            d = DatapointArray([{"t":56,"d":1}])
            d.tshift(20)
            print(d) # [{"t":76,"d":1}]


raw = self.raw()<EOL>s = <NUM_LIT:0><EOL>for i in range(len(raw)):<EOL><INDENT>s += raw[i]["<STR_LIT:d>"]<EOL><DEDENT>return s<EOL>

Gets the sum of the data portions of all datapoints within


return self.sum() / float(len(self))<EOL>

Gets the mean of the data portions of all datapoints within


def __init__(self, database_file_path, on_create=None, apikey=None, onsync=None, onsyncfail=None, syncraise=False):

self.database = apsw.Connection(database_file_path)<EOL>c = self.database.cursor()<EOL>c.execute(<EOL>"<STR_LIT>")<EOL>c.execute(<EOL>"<STR_LIT>")<EOL>c.execute(<EOL>"<STR_LIT>")<EOL>c.execute("<STR_LIT>")<EOL>row_number = next(c)[<NUM_LIT:0>]<EOL>if row_number == <NUM_LIT:0>:<EOL><INDENT>logging.debug("<STR_LIT>")<EOL>c.execute("<STR_LIT>",<EOL>(CONNECTORDB_URL, ))<EOL><DEDENT>c.execute(<EOL>"<STR_LIT>")<EOL>self.__apikey, self.__serverurl, self.__lastsync, self.__syncperiod = next(<EOL>c)<EOL>c.execute("<STR_LIT>")<EOL>self.streams = {}<EOL>for row in c.fetchall():<EOL><INDENT>self.streams[row[<NUM_LIT:0>]] = json.loads(row[<NUM_LIT:1>])<EOL><DEDENT>if apikey is not None:<EOL><INDENT>self.apikey = apikey<EOL><DEDENT>self.synclock = threading.Lock()<EOL>self.syncthread = None<EOL>self.__cdb = None<EOL>self.onsync = onsync<EOL>self.onsyncfail = onsyncfail<EOL>self.syncraise = syncraise<EOL>if on_create is not None and row_number == <NUM_LIT:0>:<EOL><INDENT>try:<EOL><INDENT>if False == on_create(self):<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>except:<EOL><INDENT>self.database.close()<EOL>os.remove(database_file_path)<EOL>raise<EOL><DEDENT><DEDENT>

Logger is started by passing its database file, and an optional callback which is run if the database
        is not initialized, allowing setup code to be only run once.

        The on_create callback can optionally be used to initialize the necessary api keys and such.
        If on_create returns False or raises an error, the uninitialized database file will be removed.


if self.__cdb is None:<EOL><INDENT>logging.debug("<STR_LIT>" + self.serverurl)<EOL>self.__cdb = ConnectorDB(self.apikey, url=self.serverurl)<EOL><DEDENT>return self.__cdb<EOL>

Returns the ConnectorDB object that the logger uses. Raises an error if Logger isn't able to connect


self.connectordb.ping()<EOL>

Attempts to ping the currently connected ConnectorDB database. Returns an error if it fails to connect


c = self.database.cursor()<EOL>c.execute("<STR_LIT>")<EOL>

Deletes all cached data without syncing it to the server


self.stop()<EOL>with self.synclock:<EOL><INDENT>self.database.close()<EOL><DEDENT>

Closes the database connections and stops all synchronization.


def addStream(self, streamname, schema=None, **kwargs):

stream = self.connectordb[streamname]<EOL>if not stream.exists():<EOL><INDENT>if schema is not None:<EOL><INDENT>stream.create(schema, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>" % (streamname, ))<EOL><DEDENT><DEDENT>self.addStream_force(streamname, stream.schema)<EOL>

Adds the given stream to the logger. Requires an active connection to the ConnectorDB database.

        If a schema is not specified, loads the stream from the database. If a schema is specified, and the stream
        does not exist, creates the stream. You can also add stream properties such as description or nickname to be added
        during creation.


def addStream_force(self, streamname, schema=None):

c = self.database.cursor()<EOL>c.execute("<STR_LIT>",<EOL>(streamname, json.dumps(schema)))<EOL>self.streams[streamname] = schema<EOL>

This function adds the given stream to the logger, but does not check with a ConnectorDB database
        to make sure that the stream exists. Use at your own risk.


if streamname not in self.streams:<EOL><INDENT>raise Exception("<STR_LIT>" % (streamname, ))<EOL><DEDENT>validate(value, self.streams[streamname])<EOL>value = json.dumps(value)<EOL>logging.debug("<STR_LIT>" % (streamname, value))<EOL>c = self.database.cursor()<EOL>c.execute("<STR_LIT>",<EOL>(streamname, time.time(), value))<EOL>

Insert the datapoint into the logger for the given stream name. The logger caches the datapoint
        and eventually synchronizes it with ConnectorDB


c = self.database.cursor()<EOL>c.execute("<STR_LIT>")<EOL>try:<EOL><INDENT>for streamname in data_dict:<EOL><INDENT>if streamname not in self.streams:<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>" % (streamname, ))<EOL><DEDENT>for dp in data_dict[streamname]:<EOL><INDENT>validate(dp["<STR_LIT:d>"], self.streams[streamname])<EOL>c.execute("<STR_LIT>",<EOL>(streamname, dp["<STR_LIT:t>"], dp["<STR_LIT:d>"]))<EOL><DEDENT><DEDENT><DEDENT>except:<EOL><INDENT>c.execute("<STR_LIT>")<EOL>raise<EOL><DEDENT>c.exectute("<STR_LIT>")<EOL>

Inserts data into the cache, if the data is a dict of the form {streamname: [{"t": timestamp,"d":data,...]}


logging.debug("<STR_LIT>")<EOL>failed = False<EOL>try:<EOL><INDENT>cdb = self.connectordb<EOL>cdb.ping()<EOL>with self.synclock:<EOL><INDENT>c = self.database.cursor()<EOL>for stream in self.streams:<EOL><INDENT>s = cdb[stream]<EOL>c.execute(<EOL>"<STR_LIT>",<EOL>(stream, ))<EOL>datapointArray = []<EOL>for dp in c.fetchall():<EOL><INDENT>datapointArray.append(<EOL>{"<STR_LIT:t>": dp[<NUM_LIT:1>],<EOL>"<STR_LIT:d>": json.loads(dp[<NUM_LIT:2>])})<EOL><DEDENT>if len(s) > <NUM_LIT:0>:<EOL><INDENT>newtime = s[-<NUM_LIT:1>]["<STR_LIT:t>"]<EOL>while (len(datapointArray) > <NUM_LIT:0> and datapointArray[<NUM_LIT:0>]["<STR_LIT:t>"] < newtime):<EOL><INDENT>logging.debug("<STR_LIT>")<EOL>datapointArray = datapointArray[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>if len(datapointArray) > <NUM_LIT:0>:<EOL><INDENT>logging.debug("<STR_LIT>" %<EOL>(stream, len(datapointArray)))<EOL>while (len(datapointArray) > DATAPOINT_INSERT_LIMIT):<EOL><INDENT>s.insert_array(<EOL>datapointArray[:DATAPOINT_INSERT_LIMIT])<EOL>datapointArray = datapointArray[<EOL>DATAPOINT_INSERT_LIMIT:]<EOL>c.execute(<EOL>"<STR_LIT>",<EOL>(stream, datapointArray[<NUM_LIT:0>]["<STR_LIT:t>"]))<EOL><DEDENT>s.insert_array(datapointArray)<EOL>c.execute(<EOL>"<STR_LIT>",<EOL>(stream, datapointArray[-<NUM_LIT:1>]["<STR_LIT:t>"]))<EOL><DEDENT><DEDENT>self.lastsynctime = time.time()<EOL>if self.onsync is not None:<EOL><INDENT>self.onsync()<EOL><DEDENT><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>falied = True<EOL>reraise = self.syncraise<EOL>if self.onsyncfail is not None:<EOL><INDENT>reraise = self.onsyncfail(e)<EOL><DEDENT>if reraise:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Attempt to sync with the ConnectorDB server


with self.synclock:<EOL><INDENT>if self.syncthread is not None:<EOL><INDENT>logging.warn(<EOL>"<STR_LIT>")<EOL>return<EOL><DEDENT><DEDENT>self.sync()  <EOL>self.__setsync()<EOL>

Start the logger background synchronization service. This allows you to not need to
        worry about syncing with ConnectorDB - you just insert into the Logger, and the Logger
        will by synced every syncperiod.


with self.synclock:<EOL><INDENT>if self.syncthread is not None:<EOL><INDENT>self.syncthread.cancel()<EOL>self.syncthread = None<EOL><DEDENT><DEDENT>

Stops the background synchronization thread


c = self.database.cursor()<EOL>c.execute("<STR_LIT>")<EOL>return next(c)[<NUM_LIT:0>]<EOL>

Returns the number of datapoints currently cached


return streamname in self.streams<EOL>

Returns whether the logger is caching the given stream name


return self.__syncperiod<EOL>

Syncperiod is the time in seconds between attempting to synchronize with ConnectorDB.
        The Logger will gather all data in its sqlite database between sync periods, and every syncperiod
        seconds, it will attempt to connect to write the data to ConnectorDB.


return self.__lastsync<EOL>

The timestamp of the most recent successful synchronization with the server


return self.__apikey<EOL>

The API key used to connect to ConnectorDB. This needs to be set before the logger can do anything!
        The apikey only needs to be set once, since it is stored in the Logger database.

        Note that changing the api key is not supported during logger runtime (after start is called).
        Logger must be recreated for a changed apikey to come into effect.


return self.__serverurl<EOL>

The URL of the ConnectorDB server that Logger is using. By default this is connectordb.com, but can
        be set with this property. Note that the property will only take into effect before runtime


c = self.database.cursor()<EOL>c.execute("<STR_LIT>")<EOL>return json.loads(next(c)[<NUM_LIT:0>])<EOL>

The data property allows the user to save settings/data in the database, so that
        there does not need to be extra code messing around with settings.

        Use this property to save things that can be converted to JSON inside the logger database,
        so that you don't have to mess with configuration files or saving setting otherwise::

            from connectordb.logger import Logger

            l = Logger("log.db")

            l.data = {"hi": 56}

            # prints the data dictionary
            print l.data


return self.connectordb.ping()<EOL>

Gets the name of the currently logged in device


value = float(value)<EOL>if value >= <NUM_LIT>:<EOL><INDENT>gigabytes = value / <NUM_LIT><EOL>size = '<STR_LIT>' % gigabytes<EOL><DEDENT>elif value >= <NUM_LIT>:<EOL><INDENT>megabytes = value / <NUM_LIT><EOL>size = '<STR_LIT>' % megabytes<EOL><DEDENT>elif value >= <NUM_LIT>:<EOL><INDENT>kilobytes = value / <NUM_LIT><EOL>size = '<STR_LIT>' % kilobytes<EOL><DEDENT>else:<EOL><INDENT>size = '<STR_LIT>' % value<EOL><DEDENT>return size<EOL>

Convert a byte value into a human-readable format.


return timedelta(<NUM_LIT:0>, int(value))<EOL>


return datetime.fromtimestamp(int(value)).strftime('<STR_LIT>')<EOL>


if not value:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if int(value):<EOL><INDENT>return _("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return _("<STR_LIT>")<EOL><DEDENT>


try:<EOL><INDENT>return round(<NUM_LIT:100> * float(data[part]) / float(data[total]), <NUM_LIT:1>)<EOL><DEDENT>except ZeroDivisionError:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>


server_info = {}<EOL>for svr in mc_client.get_stats():<EOL><INDENT>svr_info = svr[<NUM_LIT:0>].split('<STR_LIT:U+0020>')<EOL>svr_name = svr_info[<NUM_LIT:0>]<EOL>svr_stats = svr[<NUM_LIT:1>]<EOL>svr_stats['<STR_LIT>'] = _percent(svr_stats, '<STR_LIT>', '<STR_LIT>')<EOL>svr_stats['<STR_LIT>'] = _percent(svr_stats, '<STR_LIT>', '<STR_LIT>')<EOL>svr_stats['<STR_LIT>'] = _percent(svr_stats, '<STR_LIT>', '<STR_LIT>')<EOL>if server_name and server_name == svr_name:<EOL><INDENT>return svr_stats<EOL><DEDENT>server_info[svr_name] = svr_stats<EOL><DEDENT>return server_info<EOL>


server_info = {}<EOL>for svr in mc_client.get_slabs():<EOL><INDENT>svr_info = svr[<NUM_LIT:0>].split('<STR_LIT:U+0020>')<EOL>svr_name = svr_info[<NUM_LIT:0>]<EOL>if server_name and server_name == svr_name:<EOL><INDENT>return svr[<NUM_LIT:1>]<EOL><DEDENT>server_info[svr_name] = svr[<NUM_LIT:1>]<EOL><DEDENT>return server_info<EOL>


try:<EOL><INDENT>return dict(site.each_context(request).items() + data.items())<EOL><DEDENT>except AttributeError:<EOL><INDENT>return data<EOL><DEDENT>

Add admin global context, for compatibility with Django 1.7


data = {<EOL>'<STR_LIT>': _get_cache_stats(),<EOL>'<STR_LIT>': hasattr(mc_client, '<STR_LIT>'),<EOL>}<EOL>return render_to_response('<STR_LIT>', data, RequestContext(request))<EOL>


<EOL>if not isinstance(mc_client, dict):<EOL><INDENT>cache_stats = _get_cache_stats()<EOL><DEDENT>else:<EOL><INDENT>cache_stats = None<EOL><DEDENT>if cache_stats:<EOL><INDENT>data = _context_data({<EOL>'<STR_LIT:title>': _('<STR_LIT>'),<EOL>'<STR_LIT>': cache_stats,<EOL>'<STR_LIT>': hasattr(mc_client, '<STR_LIT>'),<EOL>'<STR_LIT>': SETTINGS['<STR_LIT>'],<EOL>},<EOL>request)<EOL>template = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>data = _context_data({<EOL>'<STR_LIT:title>': _('<STR_LIT>'),<EOL>'<STR_LIT>': _('<STR_LIT>'),<EOL>},<EOL>request)<EOL>template = '<STR_LIT>'<EOL><DEDENT>return render_to_response(template, data, RequestContext(request))<EOL>


server_name = server_name.strip('<STR_LIT:/>')<EOL>data = _context_data({<EOL>'<STR_LIT:title>': _('<STR_LIT>') % server_name,<EOL>'<STR_LIT>': _get_cache_stats(server_name),<EOL>},<EOL>request)<EOL>return render_to_response('<STR_LIT>', data, RequestContext(request))<EOL>


data = _context_data({<EOL>'<STR_LIT:title>': _('<STR_LIT>') % server_name,<EOL>'<STR_LIT>': _get_cache_slabs(server_name),<EOL>},<EOL>request)<EOL>return render_to_response('<STR_LIT>', data, RequestContext(request))<EOL>


mc_client.flush_all()<EOL>messages.success(request, _('<STR_LIT>'))<EOL>return redirect('<STR_LIT>')<EOL>


def safe_is_file(filename):<EOL><INDENT>try:<EOL><INDENT>return os.path.isfile(src)<EOL><DEDENT>except ValueError:  <EOL><INDENT>return False<EOL><DEDENT><DEDENT>if hasattr(src, '<STR_LIT>'):  <EOL><INDENT>data = src.read()<EOL><DEDENT>elif safe_is_file(src):<EOL><INDENT>with open(src, '<STR_LIT:rb>') as fh:<EOL><INDENT>data = fh.read()<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>data = src<EOL><DEDENT>if hasattr(data, '<STR_LIT>'):  <EOL><INDENT>exception = None<EOL>encodings = [encoding, '<STR_LIT:utf-8>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for encoding in encodings:  <EOL><INDENT>if not encoding:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>data = data.decode(encoding)<EOL>break<EOL><DEDENT>except UnicodeDecodeError as e:<EOL><INDENT>exception = e<EOL><DEDENT>except UnicodeEncodeError:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise exception  <EOL><DEDENT><DEDENT>transactions = mt940.models.Transactions()<EOL>transactions.parse(data)<EOL>return transactions<EOL>

Parses mt940 data and returns transactions object

:param src: file handler to read, filename to read or raw data as string
:return: Collection of transactions
:rtype: Transactions


def date_fixup_pre_processor(transactions, tag, tag_dict, *args):

if tag_dict['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>year = int(tag_dict['<STR_LIT>'], <NUM_LIT:10>)<EOL>_, max_month_day = calendar.monthrange(year, <NUM_LIT:2>)<EOL>if int(tag_dict['<STR_LIT>'], <NUM_LIT:10>) > max_month_day:<EOL><INDENT>tag_dict['<STR_LIT>'] = str(max_month_day)<EOL><DEDENT><DEDENT>return tag_dict<EOL>

Replace illegal February 29, 30 dates with the last day of February.

German banks use a variant of the 30/360 interest rate calculation,
where each month has always 30 days even February. Python's datetime
module won't accept such dates.


def mBank_set_transaction_code(transactions, tag, tag_dict, *args):

tag_dict['<STR_LIT>'] = int(<EOL>tag_dict[tag.slug].split('<STR_LIT:;>')[<NUM_LIT:0>].split('<STR_LIT:U+0020>', <NUM_LIT:1>)[<NUM_LIT:0>])<EOL>return tag_dict<EOL>

mBank Collect uses transaction code 911 to distinguish icoming mass
payments transactions, adding transaction_code may be helpful in further
processing


def mBank_set_iph_id(transactions, tag, tag_dict, *args):

matches = iph_id_re.search(tag_dict[tag.slug])<EOL>if matches:  <EOL><INDENT>tag_dict['<STR_LIT>'] = matches.groupdict()['<STR_LIT>']<EOL><DEDENT>return tag_dict<EOL>

mBank Collect uses ID IPH to distinguish between virtual accounts,
adding iph_id may be helpful in further processing


def mBank_set_tnr(transactions, tag, tag_dict, *args):

matches = tnr_re.search(tag_dict[tag.slug])<EOL>if matches:  <EOL><INDENT>tag_dict['<STR_LIT>'] = matches.groupdict()['<STR_LIT>']<EOL><DEDENT>return tag_dict<EOL>

mBank Collect states TNR in transaction details as unique id for
transactions, that may be used to identify the same transactions in
different statement files eg. partial mt942 and full mt940
Information about tnr uniqueness has been obtained from mBank support,
it lacks in mt940 mBank specification.


for arg in args:<EOL><INDENT>if arg is not None:<EOL><INDENT>return arg<EOL><DEDENT><DEDENT>

Return the first non-None argument

>>> coalesce()

>>> coalesce(0, 1)
0
>>> coalesce(None, 0)
0


lines = []<EOL>for line in string.splitlines():<EOL><INDENT>if strip & Strip.RIGHT:<EOL><INDENT>line = line.rstrip()<EOL><DEDENT>if strip & Strip.LEFT:<EOL><INDENT>line = line.lstrip()<EOL><DEDENT>lines.append(line)<EOL><DEDENT>return '<STR_LIT>'.join(lines)<EOL>

Join strings together and strip whitespace in between if needed


<EOL>data = '<STR_LIT:\n>'.join(self.strip(data.split('<STR_LIT:\n>')))<EOL>tag_re = re.compile(<EOL>r'<STR_LIT>',<EOL>re.MULTILINE)<EOL>matches = list(tag_re.finditer(data))<EOL>valid_matches = list(self.sanatize_tag_id_matches(matches))<EOL>for i, match in enumerate(valid_matches):<EOL><INDENT>tag_id = self.normalize_tag_id(match.group('<STR_LIT>'))<EOL>tag = self.tags.get(match.group('<STR_LIT>'))or self.tags[tag_id]<EOL>if valid_matches[i + <NUM_LIT:1>:]:<EOL><INDENT>tag_data =data[match.end():valid_matches[i + <NUM_LIT:1>].start()].strip()<EOL><DEDENT>else:<EOL><INDENT>tag_data = data[match.end():].strip()<EOL><DEDENT>tag_dict = tag.parse(self, tag_data)<EOL>for processor in self.processors.get('<STR_LIT>' % tag.slug, []):<EOL><INDENT>tag_dict = processor(self, tag, tag_dict)<EOL><DEDENT>result = tag(self, tag_dict)<EOL>for processor in self.processors.get('<STR_LIT>' % tag.slug, []):<EOL><INDENT>result = processor(self, tag, tag_dict, result)<EOL><DEDENT>if isinstance(tag, mt940.tags.Statement):<EOL><INDENT>if not self.transactions:<EOL><INDENT>transaction = Transaction(self)<EOL>self.transactions.append(transaction)<EOL><DEDENT>if transaction.data.get('<STR_LIT:id>'):<EOL><INDENT>transaction = Transaction(self, result)<EOL>self.transactions.append(transaction)<EOL><DEDENT>else:<EOL><INDENT>transaction.data.update(result)<EOL><DEDENT><DEDENT>elif issubclass(tag.scope, Transaction) and self.transactions:<EOL><INDENT>for k, v in _compat.iteritems(result):<EOL><INDENT>if k in transaction.data and hasattr(v, '<STR_LIT>'):<EOL><INDENT>transaction.data[k] += '<STR_LIT>' % v.strip()<EOL><DEDENT>else:<EOL><INDENT>transaction.data[k] = v<EOL><DEDENT><DEDENT><DEDENT>elif issubclass(tag.scope, Transactions):  <EOL><INDENT>self.data.update(result)<EOL><DEDENT><DEDENT>return self.transactions<EOL>

Parses mt940 data, expects a string with data

        Args:
            data (str): The MT940 data

        Returns: :py:class:`list` of :py:class:`Transaction`


def devpiserver_on_upload(stage, project, version, link):

if link.entry and link.entry.file_exists() and link.entry.basename.endswith('<STR_LIT>'):<EOL><INDENT>threadlog.info("<STR_LIT>", link.entry.basename)<EOL>new_version = parse_version(version)<EOL>latest_version = parse_version(stage.get_latest_version_perstage(project))<EOL>if latest_version > new_version:<EOL><INDENT>threadlog.debug("<STR_LIT>", latest_version)<EOL>return<EOL><DEDENT>metadata = extract_metadata_from_wheel_file(link.entry.file_os_path())<EOL>linkstore = stage.get_linkstore_perstage(link.project, link.version)<EOL>project_dir = '<STR_LIT>' % (linkstore.filestore.storedir, stage.name, project)<EOL>if not os.path.exists(project_dir):<EOL><INDENT>os.mkdir(project_dir)<EOL><DEDENT>json_path = '<STR_LIT>' % (project_dir, project, new_version)<EOL>with open(json_path, '<STR_LIT:w>') as fd:<EOL><INDENT>fd.write(json.dumps(metadata))<EOL><DEDENT>threadlog.info("<STR_LIT>", metadata, json_path)<EOL>symlink_path = '<STR_LIT>' % project_dir<EOL>if os.path.exists(symlink_path):<EOL><INDENT>os.unlink(symlink_path)<EOL><DEDENT>os.symlink(json_path, symlink_path)<EOL><DEDENT>

called when a file is uploaded to a private stage for
    a projectname/version.  link.entry.file_exists() may be false because
    a more recent revision deleted the file (and files are not revisioned).
    NOTE that this hook is currently NOT called for the implicit "caching"
    uploads to the pypi mirror.

    If the uploaded file is a wheel and is the latest version on this index,
    store its metadata in json file at the root of index/+f/ directory.
    With the standard config with nginx, nginx will directly serve this file.


def create_gdal_datasource(fname=None, dtype=np.ubyte, bands=<NUM_LIT:1>):

xsize, ysize = <NUM_LIT>, <NUM_LIT:1000><EOL>arr = np.ones((ysize, xsize), dtype=dtype)<EOL>if fname:<EOL><INDENT>driver = gdal.GetDriverByName('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>driver = gdal.GetDriverByName('<STR_LIT>')<EOL>fname = '<STR_LIT>'<EOL><DEDENT>datasource = driver.Create(fname, xsize, ysize, bands)<EOL>for bandnum in range(<NUM_LIT:1>, bands + <NUM_LIT:1>):<EOL><INDENT>band = datasource.GetRasterBand(bandnum)<EOL>band.WriteArray(arr)<EOL>band.SetNoDataValue(<NUM_LIT:0>)<EOL><DEDENT>sref = osr.SpatialReference()<EOL>sref.ImportFromEPSG(<NUM_LIT>)<EOL>datasource.SetProjection(sref.ExportToWkt())<EOL>geotransform = (-<NUM_LIT>, <NUM_LIT>, <NUM_LIT:0.0>, <NUM_LIT>, <NUM_LIT:0.0>, -<NUM_LIT>)<EOL>datasource.SetGeoTransform(geotransform)<EOL>datasource.FlushCache()<EOL>return datasource<EOL>


drivers = {}<EOL>for i in range(gdal.GetDriverCount()):<EOL><INDENT>d = gdal.GetDriver(i)<EOL>drivers[d.ShortName] = d.GetMetadata()<EOL><DEDENT>return drivers<EOL>

Returns a dictionary of enabled GDAL Driver metadata keyed by the
    'ShortName' attribute.


ext = (os.path.splitext(path)[<NUM_LIT:1>][<NUM_LIT:1>:] or path).lower()<EOL>drivers = drivers or ImageDriver.registry if ext else {}<EOL>for name, meta in drivers.items():<EOL><INDENT>if ext == meta.get('<STR_LIT>', '<STR_LIT>').lower():<EOL><INDENT>return ImageDriver(name)<EOL><DEDENT><DEDENT>return None<EOL>

Returns the gdal.Driver for a path or None based on the file extension.

    Arguments:
    path -- file path as str with a GDAL supported file extension


return ['<STR_LIT>' % (k, v) for k, v in d.items()]<EOL>

Returns a GDAL formatted options list of strings from a dict.


driver = ImageDriver('<STR_LIT>')<EOL>rast = driver.raster(driver.ShortName, size)<EOL>rast.affine = affine<EOL>rast.sref = geom.GetSpatialReference()<EOL>with MemoryLayer.from_records([(<NUM_LIT:1>, geom)]) as ml:<EOL><INDENT>status = gdal.RasterizeLayer(rast.ds, (<NUM_LIT:1>,), ml.layer, burn_values=(<NUM_LIT:1>,))<EOL><DEDENT>arr = rast.array()<EOL>rast.close()<EOL>return arr<EOL>

Converts an OGR polygon to a 2D NumPy array.

    Arguments:
    geom -- OGR Geometry
    size -- array size in pixels as a tuple of (width, height)
    affine -- AffineTransform


driver = ImageDriver('<STR_LIT>')<EOL>r2 = driver.raster(driver.ShortName, rast.size)<EOL>r2.affine = rast.affine<EOL>sref = rast.sref<EOL>if not sref.srid:<EOL><INDENT>sref = SpatialReference(<NUM_LIT>)<EOL><DEDENT>r2.sref = sref<EOL>ml = MemoryLayer(sref, layer.GetGeomType())<EOL>ml.load(layer)<EOL>status = gdal.RasterizeLayer(<EOL>r2.ds, (<NUM_LIT:1>,), ml.layer, options=['<STR_LIT>' % ml.id])<EOL>ml.close()<EOL>return r2<EOL>

Returns a Raster from layer features.

    Arguments:
    layer -- Layer to rasterize
    rast -- Raster with target affine, size, and sref


return zip(*np.unique(arr, return_counts=True))<EOL>

Returns a list of two-tuples with unique value and occurrence count.

    Arguments:
    arr -- numpy ndarray


path = getattr(path, '<STR_LIT:name>', path)<EOL>try:<EOL><INDENT>return Raster(vsiprefix(path), mode)<EOL><DEDENT>except AttributeError:<EOL><INDENT>try:<EOL><INDENT>imgdata = path.read()<EOL><DEDENT>except AttributeError:<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>imgio = MemFileIO(delete=False)<EOL>gdal.FileFromMemBuffer(imgio.name, imgdata)<EOL>return Raster(imgio, mode)<EOL><DEDENT><DEDENT>raise ValueError('<STR_LIT>' % path)<EOL>

Returns a Raster instance.

    Arguments:
    path -- local or remote path as str or file-like object
    Keyword args:
    mode -- gdal constant representing access mode


def frombytes(data, size, bandtype=gdal.GDT_Byte):

r = ImageDriver('<STR_LIT>').raster('<STR_LIT>', size, bandtype)<EOL>r.frombytes(data)<EOL>return r<EOL>

Returns an in-memory raster initialized from a pixel buffer.

    Arguments:
    data -- byte buffer of raw pixel data
    size -- two or three-tuple of (xsize, ysize, bandcount)
    bandtype -- band data type


def __init__(self, xorigin, xscale, rx, yorigin, ry, yscale):

<EOL>self.origin = (xorigin, yorigin)<EOL>self.scale = (xscale, yscale)<EOL>self.rotation = (rx, ry)<EOL>self._len = len(self.tuple)<EOL>

Generally this will be initialized from a six-element tuple in the
        format returned by gdal.Dataset.GetGeoTransform().

        Arguments:
        xorigin -- top left corner x coordinate
        xscale -- x scaling
        rx -- x rotation
        yorigin -- top left corner y coordinate
        ry -- y rotation
        yscale -- y scaling


geotransform = self.tuple<EOL>for x, y in coords:<EOL><INDENT>geo_x = geotransform[<NUM_LIT:0>] + geotransform[<NUM_LIT:1>] * x + geotransform[<NUM_LIT:2>] * y<EOL>geo_y = geotransform[<NUM_LIT:3>] + geotransform[<NUM_LIT:4>] * x + geotransform[<NUM_LIT:5>] * y<EOL>geo_x += geotransform[<NUM_LIT:1>] / <NUM_LIT><EOL>geo_y += geotransform[<NUM_LIT:5>] / <NUM_LIT><EOL>yield geo_x, geo_y<EOL><DEDENT>

Convert image pixel/line coordinates to georeferenced x/y, return a
        generator of two-tuples.

        Arguments:
        coords -- input coordinates as iterable containing two-tuples/lists
        such as ((0, 0), (10, 10))


<EOL>origin_x, origin_y = self.origin<EOL>sx, sy = self.scale<EOL>return [(int(math.floor((x - origin_x) / sx)),<EOL>int(math.floor((y - origin_y) / sy)))<EOL>for x, y in coords]<EOL>

Transform from projection coordinates (Xp,Yp) space to pixel/line
        (P,L) raster space, based on the provided geotransformation.

        Arguments:
        coords -- input coordinates as iterable containing two-tuples/lists
        such as ((-120, 38), (-121, 39))


def __init__(self, driver='<STR_LIT>', strictmode=True, **kwargs):

if hasattr(driver, '<STR_LIT>'):<EOL><INDENT>driver = gdal.GetDriverByName(str(driver)) or driver<EOL><DEDENT>if not isinstance(driver, gdal.Driver):<EOL><INDENT>raise TypeError('<STR_LIT>' % driver)<EOL><DEDENT>self._driver = driver<EOL>self.settings = kwargs or self.defaults.get(self.ext, {})<EOL>self.strictmode = strictmode<EOL>self._options = None<EOL>self.writable = self._writekey in self.info<EOL>self.copyable = self._copykey in self.info<EOL>

Keyword args:
driver -- str gdal.Driver name like 'GTiff' or gdal.Driver instance
kwargs -- GDAL raster creation options


if not self.copyable:<EOL><INDENT>raise IOError('<STR_LIT>')<EOL><DEDENT>if not isinstance(source, Raster):<EOL><INDENT>source = Raster(source)<EOL>should_close = True<EOL><DEDENT>else:<EOL><INDENT>should_close = False<EOL><DEDENT>if source.name == dest:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % source.name)<EOL><DEDENT>settings = driverdict_tolist(self.settings)<EOL>ds = self.CreateCopy(dest, source.ds, self.strictmode,<EOL>options=settings)<EOL>if should_close:<EOL><INDENT>source.close()<EOL><DEDENT>return Raster(ds)<EOL>

Returns a copied Raster instance.

        Arguments:
        source -- the source Raster instance or filepath as str
        dest -- destination filepath as str


if not self.writable:<EOL><INDENT>raise IOError('<STR_LIT>')<EOL><DEDENT>options = kwargs.pop('<STR_LIT>', {})<EOL>kwargs['<STR_LIT>'] = driverdict_tolist(options or self.settings)<EOL>return self._driver.Create(*args, **kwargs)<EOL>

Calls Driver.Create() with optionally provided creation options as
        dict, or falls back to driver specific defaults.


return cls._filter_by(cls._writekey)<EOL>

Return a dict of drivers supporting raster writes.


return cls._filter_by(cls._copykey)<EOL>

Return a dict of drivers supporting raster copies.


if self._options is None:<EOL><INDENT>try:<EOL><INDENT>elem = ET.fromstring(<EOL>self.info.get('<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>except ET.ParseError:<EOL><INDENT>elem = []<EOL><DEDENT>opts = {}<EOL>for child in elem:<EOL><INDENT>choices = [val.text for val in child]<EOL>if choices:<EOL><INDENT>child.attrib.update(choices=choices)<EOL><DEDENT>opts[child.attrib.pop('<STR_LIT:name>')] = child.attrib<EOL><DEDENT>self._options = opts<EOL><DEDENT>return self._options<EOL>

Returns a dict of driver specific raster creation options.

        See GDAL format docs at http://www.gdal.org/formats_list.html


try:<EOL><INDENT>return os.path.getsize(path) == <NUM_LIT:0><EOL><DEDENT>except OSError:<EOL><INDENT>return True<EOL><DEDENT>

Returns True if file is empty or non-existent.


def raster(self, path, size, bandtype=gdal.GDT_Byte):

path = getattr(path, '<STR_LIT:name>', path)<EOL>try:<EOL><INDENT>is_multiband = len(size) > <NUM_LIT:2><EOL>nx, ny, nbands = size if is_multiband else size + (<NUM_LIT:1>,)<EOL><DEDENT>except (TypeError, ValueError) as exc:<EOL><INDENT>exc.args = ('<STR_LIT>',)<EOL>raise<EOL><DEDENT>if nx < <NUM_LIT:1> or ny < <NUM_LIT:1>:<EOL><INDENT>raise ValueError('<STR_LIT>' % (size,))<EOL><DEDENT>if not self._is_empty(path):<EOL><INDENT>raise IOError('<STR_LIT>' % path)<EOL><DEDENT>ds = self.Create(path, nx, ny, nbands, bandtype)<EOL>if not ds:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % (path, str(self)))<EOL><DEDENT>return Raster(ds)<EOL>

Returns a new Raster instance.

        gdal.Driver.Create() does not support all formats.

        Arguments:
        path -- file object or path as str
        size -- two or three-tuple of (xsize, ysize, bandcount)
        bandtype -- GDAL pixel data type


return self._driver.GetMetadata()<EOL>


return self.info.get('<STR_LIT>', '<STR_LIT>')<EOL>


def __init__(self, path, mode=gdalconst.GA_ReadOnly):

if path and not isinstance(path, gdal.Dataset):<EOL><INDENT>dataset = gdal.Open(getattr(path, '<STR_LIT:name>', path), mode)<EOL><DEDENT>else:<EOL><INDENT>dataset = path<EOL><DEDENT>if not dataset:<EOL><INDENT>raise IOError('<STR_LIT>' % path)<EOL><DEDENT>self.ds = dataset<EOL>self.name = self.ds.GetDescription()<EOL>self.bandlist = range(<NUM_LIT:1>, len(self) + <NUM_LIT:1>)<EOL>self._affine = AffineTransform(*dataset.GetGeoTransform())<EOL>self._sref = SpatialReference(dataset.GetProjection())<EOL>self._nodata = None<EOL>self._envelope = None<EOL>self._driver = None<EOL>self.closed = False<EOL>dataset = None<EOL>

Initialize a Raster data set from a path or file

        Arguments:
        path -- path as str, file object, or gdal.Dataset
        Keyword args:
        mode -- gdal constant representing access mode


try:<EOL><INDENT>return getattr(self.ds, attr)<EOL><DEDENT>except AttributeError:<EOL><INDENT>if self.closed:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>raise AttributeError(<EOL>'<STR_LIT>' % (self.__class__.__name__, attr))<EOL><DEDENT>


if index < <NUM_LIT:0>:<EOL><INDENT>index += len(self)<EOL><DEDENT>index += <NUM_LIT:1><EOL>band = self.ds.GetRasterBand(index)<EOL>if not band:<EOL><INDENT>raise IndexError('<STR_LIT>' % index)<EOL><DEDENT>return band<EOL>

Returns a single Band instance.

        This is a zero-based index which matches Python list behavior but
        differs from the GDAL one-based approach of handling multiband images.


if isinstance(affine, collections.Sequence):<EOL><INDENT>affine = AffineTransform(*affine)<EOL><DEDENT>self._affine = affine<EOL>self.ds.SetGeoTransform(affine)<EOL>

Sets the affine transformation.

        Intercepts the gdal.Dataset call to ensure use as a property setter.

        Arguments:
        affine -- AffineTransform or six-tuple of geotransformation values


args = ()<EOL>if envelope:<EOL><INDENT>args = self.get_offset(envelope)<EOL><DEDENT>return self.ds.ReadAsArray(*args)<EOL>

Returns an NDArray, optionally subset by spatial envelope.

        Keyword args:
        envelope -- coordinate extent tuple or Envelope


<EOL>self.ds = None<EOL>self.closed = True<EOL>


return self._subset(geom)<EOL>

Returns a new Raster instance clipped and masked to a geometry.

        Arguments:
        geom -- OGR Polygon or MultiPolygon


return self._subset(bbox)<EOL>

Returns a new raster instance cropped to a bounding box.

        Arguments:
        bbox -- bounding box as an OGR Polygon, Envelope, or tuple


if self._envelope is None:<EOL><INDENT>origin = self.affine.origin<EOL>ur_x = origin[<NUM_LIT:0>] + self.ds.RasterXSize * self.affine.scale[<NUM_LIT:0>]<EOL>ll_y = origin[<NUM_LIT:1>] + self.ds.RasterYSize * self.affine.scale[<NUM_LIT:1>]<EOL>self._envelope = Envelope(origin[<NUM_LIT:0>], ll_y, ur_x, origin[<NUM_LIT:1>])<EOL><DEDENT>return self._envelope<EOL>

Returns the minimum bounding rectangle as a tuple of min X, min Y,
        max X, max Y.


if isinstance(envelope, collections.Sequence):<EOL><INDENT>envelope = Envelope(envelope)<EOL><DEDENT>if not (self.envelope.contains(envelope) or<EOL>self.envelope.intersects(envelope)):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>coords = self.affine.transform((envelope.ul, envelope.lr))<EOL>nxy = [(min(dest, size) - origin) or <NUM_LIT:1><EOL>for size, origin, dest in zip(self.size, *coords)]<EOL>return coords[<NUM_LIT:0>] + tuple(nxy)<EOL>

Returns a 4-tuple pixel window (x_offset, y_offset, x_size, y_size).

        Arguments:
        envelope -- coordinate extent tuple or Envelope


if self._driver is None:<EOL><INDENT>self._driver = ImageDriver(self.ds.GetDriver())<EOL><DEDENT>return self._driver<EOL>

Returns the underlying ImageDriver instance.


size = size or self.size + (len(self),)<EOL>band = self.ds.GetRasterBand(<NUM_LIT:1>)<EOL>driver = ImageDriver('<STR_LIT>')<EOL>rcopy = driver.raster(driver.ShortName, size, band.DataType)<EOL>rcopy.sref = self.GetProjection()<EOL>rcopy.affine = affine or tuple(self.affine)<EOL>colors = band.GetColorTable()<EOL>for outband in rcopy:<EOL><INDENT>if self.nodata is not None:<EOL><INDENT>outband.SetNoDataValue(self.nodata)<EOL><DEDENT>if colors:<EOL><INDENT>outband.SetColorTable(colors)<EOL><DEDENT><DEDENT>return rcopy<EOL>

Derive new Raster instances.

        Keyword args:
        size -- tuple of image size (width, height)
        affine -- AffineTransform or six-tuple of geotransformation values


if geometry is None:<EOL><INDENT>return self._masked_array()<EOL><DEDENT>geom = transform(geometry, self.sref)<EOL>env = Envelope.from_geom(geom).intersect(self.envelope)<EOL>arr = self._masked_array(env)<EOL>if geom.GetGeometryType() != ogr.wkbPoint:<EOL><INDENT>dims = self.get_offset(env)[<NUM_LIT:2>:]<EOL>affine = AffineTransform(*tuple(self.affine))<EOL>affine.origin = env.ul<EOL>mask = ~np.ma.make_mask(geom_to_array(geom, dims, affine))<EOL>arr.mask = arr.mask | mask<EOL><DEDENT>return arr<EOL>

Returns a MaskedArray using nodata values.

        Keyword args:
        geometry -- any geometry, envelope, or coordinate extent tuple


if self._nodata is None:<EOL><INDENT>self._nodata = self[<NUM_LIT:0>].GetNoDataValue()<EOL><DEDENT>return self._nodata<EOL>

Returns read only property for band nodata value, assuming single
        band rasters for now.


args = args or (<NUM_LIT:0>, <NUM_LIT:0>, self.ds.RasterXSize, self.ds.RasterYSize)<EOL>return self.ds.ReadRaster(*args, **kwargs)<EOL>

Returns raster data bytes for partial or full extent.

        Overrides gdal.Dataset.ReadRaster() with the full raster size by
        default.


def resample(self, size, interpolation=gdalconst.GRA_NearestNeighbour):

<EOL>factors = (size[<NUM_LIT:0>] / float(self.RasterXSize),<EOL>size[<NUM_LIT:1>] / float(self.RasterYSize))<EOL>affine = AffineTransform(*tuple(self.affine))<EOL>affine.scale = (affine.scale[<NUM_LIT:0>] / factors[<NUM_LIT:0>],<EOL>affine.scale[<NUM_LIT:1>] / factors[<NUM_LIT:1>])<EOL>dest = self.new(size, affine)<EOL>gdal.ReprojectImage(self.ds, dest.ds, None, None, interpolation)<EOL>return dest<EOL>

Returns a new instance resampled to provided size.

        Arguments:
        size -- tuple of x,y image dimensions


path = getattr(to, '<STR_LIT:name>', to)<EOL>if not driver and hasattr(path, '<STR_LIT>'):<EOL><INDENT>driver = driver_for_path(path, self.driver.filter_copyable())<EOL><DEDENT>elif hasattr(driver, '<STR_LIT>'):<EOL><INDENT>driver = ImageDriver(driver)<EOL><DEDENT>if driver is None or not driver.copyable:<EOL><INDENT>raise ValueError('<STR_LIT>' % path)<EOL><DEDENT>driver.copy(self, path).close()<EOL>

Save this instance to the path and format provided.

        Arguments:
        to -- output path as str, file, or MemFileIO instance
        Keyword args:
        driver -- GDAL driver name as string or ImageDriver


if not hasattr(sref, '<STR_LIT>'):<EOL><INDENT>sref = SpatialReference(sref)<EOL><DEDENT>self._sref = sref<EOL>self.ds.SetProjection(sref.ExportToWkt())<EOL>

Sets the spatial reference.

        Intercepts the gdal.Dataset call to ensure use as a property setter.

        Arguments:
        sref -- SpatialReference or any format supported by the constructor


shp = (self.ds.RasterYSize, self.ds.RasterXSize, self.ds.RasterCount)<EOL>return shp[:<NUM_LIT:2>] if shp[<NUM_LIT:2>] <= <NUM_LIT:1> else shp<EOL>

Returns a tuple of row, column, (band count if multidimensional).


return (self.ds.RasterXSize, self.ds.RasterYSize)<EOL>

Returns a 2-tuple of (width, height) in pixels.


def warp(self, to_sref, dest=None, interpolation=gdalconst.GRA_NearestNeighbour):

if not hasattr(to_sref, '<STR_LIT>'):<EOL><INDENT>to_sref = SpatialReference(to_sref)<EOL><DEDENT>dest_wkt = to_sref.ExportToWkt()<EOL>dtype = self[<NUM_LIT:0>].DataType<EOL>err_thresh = <NUM_LIT><EOL>vrt = gdal.AutoCreateWarpedVRT(self.ds, None, dest_wkt,<EOL>interpolation, err_thresh)<EOL>if vrt is None:<EOL><INDENT>raise ValueError('<STR_LIT>' % (self, dest_wkt))<EOL><DEDENT>warpsize = (vrt.RasterXSize, vrt.RasterYSize, len(self))<EOL>warptrans = vrt.GetGeoTransform()<EOL>vrt = None<EOL>if dest is None:<EOL><INDENT>imgio = MemFileIO()<EOL>rwarp = self.driver.raster(imgio, warpsize, dtype)<EOL>imgio.close()<EOL><DEDENT>else:<EOL><INDENT>rwarp = self.driver.raster(dest, warpsize, dtype)<EOL><DEDENT>rwarp.SetGeoTransform(warptrans)<EOL>rwarp.SetProjection(to_sref)<EOL>if self.nodata is not None:<EOL><INDENT>for band in rwarp:<EOL><INDENT>band.SetNoDataValue(self.nodata)<EOL>band = None<EOL><DEDENT><DEDENT>gdal.ReprojectImage(self.ds, rwarp.ds, None, None, interpolation)<EOL>return rwarp<EOL>

Returns a new reprojected instance.

        Arguments:
        to_sref -- spatial reference as a proj4 or wkt string, or a
        SpatialReference
        Keyword args:
        dest -- filepath as str
        interpolation -- GDAL interpolation type


<EOL>try:<EOL><INDENT>geom = getattr(geom, '<STR_LIT>', Envelope(geom).polygon)<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>geom.AssignSpatialReference(to_sref)<EOL><DEDENT>try:<EOL><INDENT>geom_sref = geom.GetSpatialReference()<EOL><DEDENT>except AttributeError:<EOL><INDENT>return transform(Geometry(geom), to_sref)<EOL><DEDENT>if geom_sref is None:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>if not geom_sref.IsSame(to_sref):<EOL><INDENT>geom = geom.Clone()<EOL>geom.TransformTo(to_sref)<EOL><DEDENT>return geom<EOL>

Returns a transformed Geometry.

    Arguments:
    geom -- any coercible Geometry value or Envelope
    to_sref -- SpatialReference or EPSG ID as int


<EOL>arg = kwargs.pop('<STR_LIT>', None) or len(args) and args[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>srs = kwargs.pop('<STR_LIT>', None) or arg.srs.wkt<EOL><DEDENT>except AttributeError:<EOL><INDENT>srs = SpatialReference(<NUM_LIT>)<EOL><DEDENT>if hasattr(arg, '<STR_LIT>'):<EOL><INDENT>geom = ogr.CreateGeometryFromJson(json.dumps(arg))<EOL><DEDENT>elif hasattr(arg, '<STR_LIT>'):<EOL><INDENT>char = arg[<NUM_LIT:0>] if arg else '<STR_LIT:U+0020>'<EOL>i = char if isinstance(char, int) else ord(char)<EOL>if i in (<NUM_LIT:0>, <NUM_LIT:1>):<EOL><INDENT>geom = ogr.CreateGeometryFromWkb(arg)<EOL><DEDENT>elif arg.startswith('<STR_LIT:{>'):<EOL><INDENT>geom = ogr.CreateGeometryFromJson(arg)<EOL><DEDENT>elif arg.startswith('<STR_LIT>'):<EOL><INDENT>geom = ogr.CreateGeometryFromGML(arg)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % arg)<EOL><DEDENT><DEDENT>elif hasattr(arg, '<STR_LIT>'):<EOL><INDENT>geom = ogr.CreateGeometryFromWkb(bytes(arg.wkb))<EOL><DEDENT>else:<EOL><INDENT>geom = ogr.Geometry(*args, **kwargs)<EOL><DEDENT>if geom:<EOL><INDENT>if not isinstance(srs, SpatialReference):<EOL><INDENT>srs = SpatialReference(srs)<EOL><DEDENT>geom.AssignSpatialReference(srs)<EOL><DEDENT>return geom<EOL>

Returns an ogr.Geometry instance optionally created from a geojson str
    or dict. The spatial reference may also be provided.


if len(args) == <NUM_LIT:1>:<EOL><INDENT>args = args[<NUM_LIT:0>]<EOL><DEDENT>try:<EOL><INDENT>extent = list(map(float, args))<EOL><DEDENT>except (TypeError, ValueError) as exc:<EOL><INDENT>exc.args = ('<STR_LIT>' % repr(args),)<EOL>raise<EOL><DEDENT>try:<EOL><INDENT>self.min_x, self.max_x = sorted(extent[::<NUM_LIT:2>])<EOL>self.min_y, self.max_y = sorted(extent[<NUM_LIT:1>::<NUM_LIT:2>])<EOL><DEDENT>except ValueError as exc:<EOL><INDENT>exc.args = ('<STR_LIT>' % len(args),)<EOL>raise<EOL><DEDENT>

Creates an envelope from lower-left and upper-right coordinates.

        Arguments:
        args -- min_x, min_y, max_x, max_y or a four-tuple


return self.min_x + self.width * <NUM_LIT:0.5>, self.min_y + self.height * <NUM_LIT:0.5><EOL>

Returns the envelope centroid as a (x, y) tuple.


try:<EOL><INDENT>return (self.min_x <= other.min_x and<EOL>self.min_y <= other.min_y and<EOL>self.max_x >= other.max_x and<EOL>self.max_y >= other.max_y)<EOL><DEDENT>except AttributeError:<EOL><INDENT>return self.contains(Envelope(other))<EOL><DEDENT>

Returns true if this envelope contains another.

        Arguments:
        other -- Envelope or tuple of (minX, minY, maxX, maxY)


if len(other) == <NUM_LIT:2>:<EOL><INDENT>other += other<EOL><DEDENT>mid = len(other) // <NUM_LIT:2><EOL>self.ll = map(min, self.ll, other[:mid])<EOL>self.ur = map(max, self.ur, other[mid:])<EOL>

Expands this envelope by the given Envelope or tuple.

        Arguments:
        other -- Envelope, two-tuple, or four-tuple


extent = geom.GetEnvelope()<EOL>return Envelope(map(extent.__getitem__, (<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>, <NUM_LIT:3>)))<EOL>

Returns an Envelope from an OGR Geometry.


inter = Envelope(tuple(self))<EOL>if inter.intersects(other):<EOL><INDENT>mid = len(other) // <NUM_LIT:2><EOL>inter.ll = map(max, inter.ll, other[:mid])<EOL>inter.ur = map(min, inter.ur, other[mid:])<EOL><DEDENT>else:<EOL><INDENT>inter.ll = (<NUM_LIT:0>, <NUM_LIT:0>)<EOL>inter.ur = (<NUM_LIT:0>, <NUM_LIT:0>)<EOL><DEDENT>return inter<EOL>

Returns the intersection of this and another Envelope.


try:<EOL><INDENT>return (self.min_x <= other.max_x and<EOL>self.max_x >= other.min_x and<EOL>self.min_y <= other.max_y and<EOL>self.max_y >= other.min_y)<EOL><DEDENT>except AttributeError:<EOL><INDENT>return self.intersects(Envelope(other))<EOL><DEDENT>

Returns true if this envelope intersects another.

        Arguments:
        other -- Envelope or tuple of (minX, minY, maxX, maxY)


return self.min_x, self.min_y<EOL>


self.min_x, self.min_y = coord<EOL>


return self.max_x, self.min_y<EOL>


yfactor = xfactor if yfactor is None else yfactor<EOL>x, y = self.centroid<EOL>xshift = self.width * xfactor * <NUM_LIT:0.5><EOL>yshift = self.height * yfactor * <NUM_LIT:0.5><EOL>return Envelope(x - xshift, y - yshift, x + xshift, y + yshift)<EOL>

Returns a new envelope rescaled from center by the given factor(s).

        Arguments:
        xfactor -- int or float X scaling factor
        yfactor -- int or float Y scaling factor


ring = ogr.Geometry(ogr.wkbLinearRing)<EOL>for coord in self.ll, self.lr, self.ur, self.ul, self.ll:<EOL><INDENT>ring.AddPoint_2D(*coord)<EOL><DEDENT>polyg = ogr.Geometry(ogr.wkbPolygon)<EOL>polyg.AddGeometryDirectly(ring)<EOL>return polyg<EOL>

Returns an OGR Geometry for this envelope.


return self.ll + self.ur<EOL>


return self.min_x, self.max_y<EOL>


return self.max_x, self.max_y<EOL>


self.max_x, self.max_y = coord<EOL>


env = Envelope(bbox)<EOL>for z in zlevs:<EOL><INDENT>corners = [to_tile(*coord + (z,)) for coord in (env.ul, env.lr)]<EOL>xs, ys = [range(p1, p2 + <NUM_LIT:1>) for p1, p2 in zip(*corners)]<EOL>for coord in itertools.product(xs, ys, (z,)):<EOL><INDENT>yield coord<EOL><DEDENT><DEDENT>

Yields tile (x, y, z) tuples for a bounding box and zoom levels.

    Arguments:
    bbox - bounding box as a 4-length sequence
    zlevs - sequence of tile zoom levels


n = <NUM_LIT> ** zoom<EOL>lon = xtile / n * <NUM_LIT> - <NUM_LIT><EOL>try:<EOL><INDENT>lat_rad = math.atan(math.sinh(math.pi * (<NUM_LIT:1> - <NUM_LIT:2> * ytile / n)))<EOL><DEDENT>except OverflowError:<EOL><INDENT>raise ValueError('<STR_LIT>' % zoom)<EOL><DEDENT>lat = math.degrees(lat_rad)<EOL>return lon, lat<EOL>

Returns a tuple of (longitude, latitude) from a map tile xyz coordinate.

    See http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Lon..2Flat._to_tile_numbers_2

    Arguments:
    xtile - x tile location as int or float
    ytile - y tile location as int or float
    zoom - zoom level as int or float


lat_rad = math.radians(lat)<EOL>n = <NUM_LIT> ** zoom<EOL>xtile = int((lon + <NUM_LIT>) / <NUM_LIT> * n)<EOL>ytile = int((<NUM_LIT:1.0> - math.log(math.tan(lat_rad) +<EOL>(<NUM_LIT:1> / math.cos(lat_rad))) / math.pi) / <NUM_LIT> * n)<EOL>return xtile, ytile<EOL>

Returns a tuple of (xtile, ytile) from a (longitude, latitude) coordinate.

    See http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames

    Arguments:
    lon - longitude as int or float
    lat - latitude as int or float
    zoom - zoom level as int or float


epsg_id = (self.GetAuthorityCode('<STR_LIT>') or<EOL>self.GetAuthorityCode('<STR_LIT>'))<EOL>try:<EOL><INDENT>return int(epsg_id)<EOL><DEDENT>except TypeError:<EOL><INDENT>return<EOL><DEDENT>

Returns the EPSG ID as int if it exists.


return self.ExportToWkt()<EOL>


return self.ExportToProj4()<EOL>

Returns this projection as a proj4 string.


vpath = path.lower()<EOL>scheme = VSI_SCHEMES.get(urlparse(vpath).scheme, '<STR_LIT>')<EOL>for ext in VSI_TYPES:<EOL><INDENT>if ext in vpath:<EOL><INDENT>filesys = VSI_TYPES[ext]<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>filesys = '<STR_LIT>'<EOL><DEDENT>if filesys and scheme:<EOL><INDENT>filesys = filesys[:-<NUM_LIT:1>]<EOL><DEDENT>return '<STR_LIT>'.join((filesys, scheme, path))<EOL>

Returns a GDAL virtual filesystem prefixed path.

    Arguments:
    path -- file path as str


download(url, download_path)<EOL>

Downloads the file at the specified url to the download path.
:param url: The download url.
:param download_path: The path to the file where to store the download.
:return:


_download(URL, os.path.join(dataset_base_dir, "<STR_LIT>"))<EOL>

Download the dataset to the download dataset base dir.
:param dataset_base_dir: The directory where the root of your datalake is.
:return:


def crop_image(img, start_y, start_x, h, w):

return img[start_y:start_y + h, start_x:start_x + w, :].copy()<EOL>

Crop an image given the top left corner.
:param img: The image
:param start_y: The top left corner y coord
:param start_x: The top left corner x coord
:param h: The result height
:param w: The result width
:return: The cropped image.


for k in label.keys():<EOL><INDENT>if k.startswith("<STR_LIT>"):<EOL><INDENT>detections = label[k]<EOL>for detection in detections:<EOL><INDENT>detection.move_image(-dx, -dy)<EOL><DEDENT><DEDENT><DEDENT>

Move detections in direction dx, dy.

:param label: The label dict containing all detection lists.
:param dy: The delta in y direction as a number.
:param dx: The delta in x direction as a number.
:return:


for k in label.keys():<EOL><INDENT>if k.startswith("<STR_LIT>"):<EOL><INDENT>detections = label[k]<EOL>for detection in detections:<EOL><INDENT>detection.cx = w - detection.cx<EOL>if k == "<STR_LIT>":<EOL><INDENT>detection.theta = math.pi - detection.theta<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Horizontally flip detections according to an image flip.

:param label: The label dict containing all detection lists.
:param w: The width of the image as a number.
:return:


def augment_detections(hyper_params, feature, label):

<EOL>if hyper_params.problem.get("<STR_LIT>", None) is None:<EOL><INDENT>return feature, label<EOL><DEDENT>img_h, img_w, img_c = feature["<STR_LIT:image>"].shape<EOL>augmented_feature = {}<EOL>augmented_label = {}<EOL>augmented_feature["<STR_LIT:image>"] = feature["<STR_LIT:image>"].copy()<EOL>if "<STR_LIT>" in feature:<EOL><INDENT>augmented_feature["<STR_LIT>"] = feature["<STR_LIT>"].copy()<EOL><DEDENT>if "<STR_LIT>" in feature:<EOL><INDENT>augmented_feature["<STR_LIT>"] = feature["<STR_LIT>"]<EOL><DEDENT>augmented_feature["<STR_LIT>"] = np.array([<NUM_LIT:0>], dtype=np.uint8)<EOL>augmented_feature["<STR_LIT>"] = np.array([<NUM_LIT:0>, <NUM_LIT:0>], dtype=np.int8)<EOL>for k in label.keys():<EOL><INDENT>augmented_label[k] = [detection.copy() for detection in label[k]]<EOL><DEDENT>if hyper_params.problem.augmentation.get("<STR_LIT>", False):<EOL><INDENT>if random.random() < <NUM_LIT:0.5>:<EOL><INDENT>img_h, img_w, img_c = augmented_feature["<STR_LIT:image>"].shape<EOL>augmented_feature["<STR_LIT:image>"] = np.fliplr(augmented_feature["<STR_LIT:image>"])<EOL>if "<STR_LIT>" in feature:<EOL><INDENT>augmented_feature["<STR_LIT>"] = np.fliplr(augmented_feature["<STR_LIT>"])<EOL><DEDENT>augmented_feature["<STR_LIT>"][<NUM_LIT:0>] = <NUM_LIT:1><EOL>hflip_detections(augmented_label, img_w)<EOL><DEDENT><DEDENT>if hyper_params.problem.augmentation.get("<STR_LIT>", False):<EOL><INDENT>img_h, img_w, img_c = augmented_feature["<STR_LIT:image>"].shape<EOL>dx = int(random.random() * <NUM_LIT:3>)<EOL>dy = int(random.random() * <NUM_LIT:3>)<EOL>augmented_feature["<STR_LIT:image>"] = crop_image(augmented_feature["<STR_LIT:image>"], dy, dx, img_h - dy, img_w - dx)<EOL>if "<STR_LIT>" in feature:<EOL><INDENT>augmented_feature["<STR_LIT>"] = crop_image(augmented_feature["<STR_LIT>"], dy, dx, img_h - dy, img_w - dx)<EOL><DEDENT>augmented_feature["<STR_LIT>"][<NUM_LIT:0>] += dy<EOL>augmented_feature["<STR_LIT>"][<NUM_LIT:1>] += dx<EOL>move_detections(augmented_label, -dy, -dx)<EOL><DEDENT>if hyper_params.problem.augmentation.get("<STR_LIT>", None) is not None:<EOL><INDENT>img_h, img_w, img_c = augmented_feature["<STR_LIT:image>"].shape<EOL>target_w = hyper_params.problem.augmentation.random_crop.shape.width<EOL>target_h = hyper_params.problem.augmentation.random_crop.shape.height<EOL>delta_x = max(int(math.ceil((target_w + <NUM_LIT:1> - img_w) / <NUM_LIT:2>)), <NUM_LIT:0>)<EOL>delta_y = max(int(math.ceil((target_h + <NUM_LIT:1> - img_h) / <NUM_LIT:2>)), <NUM_LIT:0>)<EOL>move_detections(augmented_label, delta_y, delta_x)<EOL>augmented_feature["<STR_LIT:image>"] = cv2.copyMakeBorder(augmented_feature["<STR_LIT:image>"],<EOL>delta_y, delta_y, delta_x, delta_x,<EOL>cv2.BORDER_CONSTANT)<EOL>img_h, img_w, img_c = augmented_feature["<STR_LIT:image>"].shape<EOL>start_x = <NUM_LIT:0><EOL>start_y = <NUM_LIT:0><EOL>if len(augmented_label["<STR_LIT>"]) != <NUM_LIT:0>:<EOL><INDENT>idx = random.randint(<NUM_LIT:0>, len(augmented_label["<STR_LIT>"]) - <NUM_LIT:1>)<EOL>detection = augmented_label["<STR_LIT>"][idx]<EOL>start_x = int(detection.cx - random.random() * (target_w - <NUM_LIT:20>) / <NUM_LIT> - <NUM_LIT:10>)<EOL>start_y = int(detection.cy - random.random() * (target_h - <NUM_LIT:20>) / <NUM_LIT> - <NUM_LIT:10>)<EOL><DEDENT>else:<EOL><INDENT>start_x = int(img_w * random.random())<EOL>start_y = int(img_h * random.random())<EOL><DEDENT>if start_x < <NUM_LIT:0>:<EOL><INDENT>start_x = <NUM_LIT:0><EOL><DEDENT>if start_y < <NUM_LIT:0>:<EOL><INDENT>start_y = <NUM_LIT:0><EOL><DEDENT>if start_x >= img_w - target_w:<EOL><INDENT>start_x = img_w - target_w - <NUM_LIT:1><EOL><DEDENT>if start_y >= img_h - target_h:<EOL><INDENT>start_y = img_h - target_h - <NUM_LIT:1><EOL><DEDENT>augmented_feature["<STR_LIT:image>"] = crop_image(augmented_feature["<STR_LIT:image>"], start_y, start_x, target_h, target_w)<EOL>if "<STR_LIT>" in feature:<EOL><INDENT>augmented_feature["<STR_LIT>"] = crop_image(augmented_feature["<STR_LIT>"], start_y, start_x, target_h, target_w)<EOL><DEDENT>augmented_feature["<STR_LIT>"][<NUM_LIT:0>] += start_y<EOL>augmented_feature["<STR_LIT>"][<NUM_LIT:1>] += start_x<EOL>move_detections(augmented_label, -start_y, -start_x)<EOL><DEDENT>if hyper_params.problem.augmentation.get("<STR_LIT>", False):<EOL><INDENT>if random.random() < <NUM_LIT:0.5>:<EOL><INDENT>augmented_feature["<STR_LIT:image>"] = full_texture_augmentation(augmented_feature["<STR_LIT:image>"])<EOL><DEDENT><DEDENT>return augmented_feature, augmented_label<EOL>

Augment the detection dataset.

In your hyper_parameters.problem.augmentation add configurations to enable features.
Supports "enable_horizontal_flip", "enable_micro_translation", "random_crop" : {"shape": { "width", "height" }}
and "enable_texture_augmentation". Make sure to also set the "steps" otherwise this method will not be used properly.

Random crop ensures at least one detection is in the crop region.

Sample configuration
"problem": {
    "augmentation": {
        "steps": 40,
        "enable_texture_augmentation": true,
        "enable_micro_translation": true,
        "enable_horizontal_flip": true,
        "random_crop": {
            "shape": {
                "width": 256,
                "height": 256
            }
        }
    }
}

:param hyper_params: The hyper parameters object
:param feature: A dict containing all features, must be in the style created by detection datasets.
:param label: A label dict in the detection dataset style.
:return: Modified feature and label dict (copied & modified).


return []<EOL>

Converts the detection into an array.
Values depend on the data type but they are in the order specified in __slots__.
The rough structure is:
1 Class id (int)
2 Center position
3 Dimensions of object
4 Orientation of object
:return: An array containing the values.


return dict(zip(self.__slots__, self.to_array()))<EOL>

Converts the object into a dictionary.
:return: The dictionary containing all variables of a detection.


return <NUM_LIT:0.0><EOL>

Calculates the iou between two detections.
:param other: The other detection that is used to compute the iou.
:return: The intersection over union value.


@abc.abstractmethod<EOL><INDENT>def visualize(self, image, color):<DEDENT>

return<EOL>

Draw the detection into the given image.

:param image: A numpy nd array representing the image (shape=[h,w,3]).
:param color: The color to use for drawing.
:return: Nothing the input image is modified.


return<EOL>


@abc.abstractmethod<EOL><INDENT>def move_image(self, dx, dy):<DEDENT>

return<EOL>

Move the image the annotation is according to by some pixels.

:param dx: The amount of pixels the origin of the image is moved to the right.
:param dy: The amount of pixels the origin of the image is moved to the bottom.
:return:


def to_currency(self, val, currency='<STR_LIT>', cents=True, separator='<STR_LIT:U+002C>',<EOL>adjective=False):

left, right, is_negative = parse_currency_parts(val)<EOL>try:<EOL><INDENT>cr1, cr2 = self.CURRENCY_FORMS[currency]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise NotImplementedError(<EOL>'<STR_LIT>' %<EOL>(currency, self.__class__.__name__))<EOL><DEDENT>if adjective and currency in self.CURRENCY_ADJECTIVES:<EOL><INDENT>cr1 = prefix_currency(<EOL>self.CURRENCY_ADJECTIVES[currency],<EOL>cr1<EOL>)<EOL><DEDENT>minus_str = "<STR_LIT>" % self.negword if is_negative else "<STR_LIT>"<EOL>cents_str = self._cents_verbose(right, currency)if cents else self._cents_terse(right, currency)<EOL>return u'<STR_LIT>' % (<EOL>minus_str,<EOL>self.to_cardinal(left, feminine=cr1[-<NUM_LIT:1>]),<EOL>self.pluralize(left, cr1),<EOL>separator,<EOL>cents_str,<EOL>self.pluralize(right, cr2)<EOL>)<EOL>

Args:
    val: Numeric value
    currency (str): Currency code
    cents (bool): Verbose cents
    separator (str): Cent separator
    adjective (bool): Prefix currency name with adjective
Returns:
    str: Formatted string


return forms[<NUM_LIT:0>]<EOL>

:param n:
:param forms:
:return:

gettext form is nplurals=2; plural=(n != 1);
but this claims https://onzetaal.nl/taaladvies/euro-euros/
not sure if it's applied only to euro


def select_text(text, reading=False, prefer=None):

<EOL>if reading:<EOL><INDENT>text = text[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>text = text[<NUM_LIT:0>]<EOL><DEDENT>if not isinstance(text, strtype):<EOL><INDENT>common = set(text) & set(prefer or set())<EOL>if len(common) == <NUM_LIT:1>:<EOL><INDENT>text = common.pop()<EOL><DEDENT>else:<EOL><INDENT>text = text[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>return text<EOL>

Select the correct text from the Japanese number, reading and
    alternatives


ltext, lnum = lpair<EOL>rtext, rnum = rpair<EOL>if lnum > rnum:<EOL><INDENT>raise ValueError<EOL><DEDENT>if rpair == ("<STR_LIT>", <NUM_LIT:100>):<EOL><INDENT>if lpair == ("<STR_LIT>", <NUM_LIT:3>):<EOL><INDENT>rtext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:6>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL>rtext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:8>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL>rtext = "<STR_LIT>"<EOL><DEDENT><DEDENT>elif rpair == ("<STR_LIT>", <NUM_LIT:1000>):<EOL><INDENT>if lpair == ("<STR_LIT>", <NUM_LIT:3>):<EOL><INDENT>rtext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:8>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT><DEDENT>elif rpair == ("<STR_LIT>", <NUM_LIT:10>**<NUM_LIT:12>):<EOL><INDENT>if lpair == ("<STR_LIT>", <NUM_LIT:1>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:8>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:10>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT><DEDENT>elif rpair == ("<STR_LIT>", <NUM_LIT:10>**<NUM_LIT:16>):<EOL><INDENT>if lpair == ("<STR_LIT>", <NUM_LIT:1>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:6>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:8>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:10>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT>elif lpair == ("<STR_LIT>", <NUM_LIT:100>):<EOL><INDENT>ltext = "<STR_LIT>"<EOL><DEDENT><DEDENT>return ("<STR_LIT>" % (ltext, rtext), lnum * rnum)<EOL>

Merge lpair < rpair while applying semi-irregular rendaku rules


if num_str.startswith('<STR_LIT:->'):<EOL><INDENT>return '<STR_LIT>' % self.negword, num_str[<NUM_LIT:1>:]<EOL><DEDENT>return '<STR_LIT>', num_str<EOL>

Detach minus and return it as symbol with new num_str.


raise NotImplementedError<EOL>

Should resolve gettext form:
http://docs.translatehouse.org/projects/localization-guide/en/latest/l10n/pluralforms.html


left, right, is_negative = parse_currency_parts(val)<EOL>try:<EOL><INDENT>cr1, cr2 = self.CURRENCY_FORMS[currency]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise NotImplementedError(<EOL>'<STR_LIT>' %<EOL>(currency, self.__class__.__name__))<EOL><DEDENT>if adjective and currency in self.CURRENCY_ADJECTIVES:<EOL><INDENT>cr1 = prefix_currency(self.CURRENCY_ADJECTIVES[currency], cr1)<EOL><DEDENT>minus_str = "<STR_LIT>" % self.negword if is_negative else "<STR_LIT>"<EOL>cents_str = self._cents_verbose(right, currency)if cents else self._cents_terse(right, currency)<EOL>return u'<STR_LIT>' % (<EOL>minus_str,<EOL>self.to_cardinal(left),<EOL>self.pluralize(left, cr1),<EOL>separator,<EOL>cents_str,<EOL>self.pluralize(right, cr2)<EOL>)<EOL>


blocks = ()<EOL>length = len(number)<EOL>if length < <NUM_LIT:3>:<EOL><INDENT>blocks += ((number,),)<EOL><DEDENT>else:<EOL><INDENT>len_of_first_block = length % <NUM_LIT:3><EOL>if len_of_first_block > <NUM_LIT:0>:<EOL><INDENT>first_block = number[<NUM_LIT:0>:len_of_first_block],<EOL>blocks += first_block,<EOL><DEDENT>for i in range(len_of_first_block, length, <NUM_LIT:3>):<EOL><INDENT>next_block = (number[i:i + <NUM_LIT:3>],),<EOL>blocks += next_block<EOL><DEDENT><DEDENT>return blocks<EOL>

starting here, it groups the number by three from the tail
'1234567' -> (('1',),('234',),('567',))
:param number:str
:rtype:tuple


word_blocks = ()<EOL>first_block = blocks[<NUM_LIT:0>]<EOL>if len(first_block[<NUM_LIT:0>]) == <NUM_LIT:1>:<EOL><INDENT>if first_block[<NUM_LIT:0>] == '<STR_LIT:0>':<EOL><INDENT>spelling = ['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>spelling = self.BASE[int(first_block[<NUM_LIT:0>])]<EOL><DEDENT><DEDENT>elif len(first_block[<NUM_LIT:0>]) == <NUM_LIT:2>:<EOL><INDENT>spelling = self.puluh(first_block[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>spelling = (<EOL>self.ratus(first_block[<NUM_LIT:0>][<NUM_LIT:0>]) + self.puluh(first_block[<NUM_LIT:0>][<NUM_LIT:1>:<NUM_LIT:3>])<EOL>)<EOL><DEDENT>word_blocks += (first_block[<NUM_LIT:0>], spelling),<EOL>for block in blocks[<NUM_LIT:1>:]:<EOL><INDENT>spelling = self.ratus(block[<NUM_LIT:0>][<NUM_LIT:0>]) + self.puluh(block[<NUM_LIT:0>][<NUM_LIT:1>:<NUM_LIT:3>])<EOL>block += spelling,<EOL>word_blocks += block,<EOL><DEDENT>return word_blocks<EOL>

it adds the list of spelling to the blocks
(
('1',),('034',)) -> (('1',['satu']),('234',['tiga', 'puluh', 'empat'])
)
:param blocks: tuple
:rtype: tuple


def join(self, word_blocks, float_part):

word_list = []<EOL>length = len(word_blocks) - <NUM_LIT:1><EOL>first_block = word_blocks[<NUM_LIT:0>],<EOL>start = <NUM_LIT:0><EOL>if length == <NUM_LIT:1> and first_block[<NUM_LIT:0>][<NUM_LIT:0>] == '<STR_LIT:1>':<EOL><INDENT>word_list += ['<STR_LIT>']<EOL>start = <NUM_LIT:1><EOL><DEDENT>for i in range(start, length + <NUM_LIT:1>, <NUM_LIT:1>):<EOL><INDENT>word_list += word_blocks[i][<NUM_LIT:1>]<EOL>if not word_blocks[i][<NUM_LIT:1>]:<EOL><INDENT>continue<EOL><DEDENT>if i == length:<EOL><INDENT>break<EOL><DEDENT>word_list += [self.TENS_TO[(length - i) * <NUM_LIT:3>]]<EOL><DEDENT>return '<STR_LIT:U+0020>'.join(word_list) + float_part<EOL>

join the words by first join lists in the tuple
:param word_blocks: tuple
:rtype: str


version = '<STR_LIT>'<EOL>with open(fname, '<STR_LIT:r>', encoding="<STR_LIT:utf-8>") as fp:<EOL><INDENT>reg = re.compile(r'<STR_LIT>')<EOL>for line in fp:<EOL><INDENT>m = reg.match(line)<EOL>if m:<EOL><INDENT>version = m.group(<NUM_LIT:1>)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if not version:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>return version<EOL>

Parse file & return version number matching 0.0.1 regex
    Returns str or raises RuntimeError


soup = soup.find_all('<STR_LIT>', class_='<STR_LIT>')<EOL>vns = []<EOL>for item in soup[<NUM_LIT:1>:]:<EOL><INDENT>vns.append({'<STR_LIT:name>': item.string, '<STR_LIT:id>': item.a.get('<STR_LIT>')[<NUM_LIT:1>:]})<EOL><DEDENT>return vns<EOL>

Parse Visual Novel search pages.

:param soup: The BS4 class object
:return:  A list of dictionaries containing a name and id.


soup = list(soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>].children)[<NUM_LIT:1>:]<EOL>releases = []<EOL>for item in soup:<EOL><INDENT>child = list(item.children)<EOL>temp_rel = {'<STR_LIT:date>': None, '<STR_LIT>': None, '<STR_LIT>': None, '<STR_LIT:name>': None}<EOL>temp_rel['<STR_LIT:date>'] = child[<NUM_LIT:0>].string<EOL>temp_rel['<STR_LIT>'] = child[<NUM_LIT:1>].string<EOL>temp_rel['<STR_LIT>'] = child[<NUM_LIT:2>].abbr.get('<STR_LIT:title>')<EOL>temp_rel['<STR_LIT:name>'] = child[<NUM_LIT:3>].a.string<EOL>releases.append(temp_rel)<EOL>del temp_rel<EOL><DEDENT>return releases<EOL>

Parse Releases search pages.

:param soup: The BS4 class object
:return: A list of dictionaries containing a release dictionary. This is the same as the one returned in get_novel.
         It contains a Date released, Platform, Ages group and Name.


soup = soup.find_all('<STR_LIT>')<EOL>producers = []<EOL>for item in soup:<EOL><INDENT>producers.append({'<STR_LIT>': item.abbr.get('<STR_LIT:title>'), '<STR_LIT:name>': item.a.string})<EOL><DEDENT>return producers<EOL>

Parse a page of producer or staff results

:param soup: The BS4 class object
:return: A list of dictionaries containing a name and nationality.


soup = list(soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>].children)[<NUM_LIT:1>:]<EOL>characters = []<EOL>for item in soup:<EOL><INDENT>temp_c = {'<STR_LIT>': None, '<STR_LIT:name>': None, '<STR_LIT>': {}}<EOL>temp_c['<STR_LIT>'] = item.abbr.get('<STR_LIT:title>')<EOL>temp_c['<STR_LIT:name>'] = list(item.children)[<NUM_LIT:1>].a.string<EOL>temp_c['<STR_LIT>'] = []<EOL>for game in list(list(list(item.children)[<NUM_LIT:1>].children)[<NUM_LIT:1>].children):<EOL><INDENT>if isinstance(game, NavigableString):<EOL><INDENT>continue<EOL><DEDENT>temp_c['<STR_LIT>'].append({'<STR_LIT:name>': game.string, '<STR_LIT:id>': game.get('<STR_LIT>').split('<STR_LIT:/>')[<NUM_LIT:1>]})<EOL><DEDENT>characters.append(temp_c)<EOL>del temp_c<EOL><DEDENT>return characters<EOL>

Parse a page of character results.

:param soup: The BS4 class object
:return: Returns a list of dictionaries containing a name, gender and list of dictionaries containing a game name/id pair
         for games they appeared in.


soup = soup.find_all('<STR_LIT>', class_='<STR_LIT>')<EOL>tags = []<EOL>for item in soup:<EOL><INDENT>tags.append(item.a.string)<EOL><DEDENT>return tags<EOL>

Parse a page of tag or trait results. Same format.

:param soup: BS4 Class Object
:return: A list of tags, Nothing else really useful there


soup = list(soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>].children)[<NUM_LIT:1>:]<EOL>users = []<EOL>for item in soup:<EOL><INDENT>t_u = {'<STR_LIT:name>': None, '<STR_LIT>': None}<EOL>t_u['<STR_LIT:name>'] = list(item.children)[<NUM_LIT:0>].a.string<EOL>t_u['<STR_LIT>'] = list(item.children)[<NUM_LIT:1>].string<EOL>users.append(t_u)<EOL>del t_u<EOL><DEDENT>return users<EOL>

Parse a page of user results

:param soup: Bs4 Class object
:return: A list of dictionaries containing a name and join date


async def search_vndb(self, stype, term):

fstype = "<STR_LIT>"<EOL>if stype not in ['<STR_LIT:v>', '<STR_LIT:r>', '<STR_LIT:p>', '<STR_LIT:s>', '<STR_LIT:c>', '<STR_LIT:g>', '<STR_LIT:i>', '<STR_LIT:u>']:<EOL><INDENT>raise VNDBBadStype(stype)<EOL><DEDENT>else:<EOL><INDENT>if stype in ['<STR_LIT:v>', '<STR_LIT:p>', '<STR_LIT:s>', '<STR_LIT:c>', '<STR_LIT:u>']:<EOL><INDENT>fstype = '<STR_LIT>'.format(stype)<EOL><DEDENT>elif stype in ['<STR_LIT:g>', '<STR_LIT:i>']:<EOL><INDENT>fstype = '<STR_LIT>'.format(stype)<EOL><DEDENT>elif stype == '<STR_LIT:r>':<EOL><INDENT>fstype = '<STR_LIT>'<EOL><DEDENT><DEDENT>async with self.session.get(self.base_url + "<STR_LIT:{}>".format(fstype), params={"<STR_LIT:q>": term}, headers=self.headers) as response:<EOL><INDENT>if response.status == <NUM_LIT>:<EOL><INDENT>raise aiohttp.HttpBadRequest("<STR_LIT>")<EOL><DEDENT>elif '<STR_LIT>' not in response.url:<EOL><INDENT>raise VNDBOneResult(term, response.url.rsplit('<STR_LIT:/>', <NUM_LIT:1>)[<NUM_LIT:1>])<EOL><DEDENT>text = await response.text()<EOL>if '<STR_LIT>' in text:<EOL><INDENT>raise VNDBNoResults(term)<EOL><DEDENT>soup = BeautifulSoup(text, '<STR_LIT>')<EOL>resp = await self.parse_search(stype, soup)<EOL>if resp == []:<EOL><INDENT>raise VNDBNoResults(term)<EOL><DEDENT>return resp<EOL><DEDENT>

Search vndb.org for a term and return matching results from type.

:param stype: type to search for.
    Type should be one of:
        v - Visual Novels
        r - Releases
        p - Producers
        s - Staff
        c - Characters
        g - Tags
        i - traits
        u - Users
:param term: string to search for
:return: Results. Result format depends on what you searched for. See the Parsing.py module for more specific documentation.

Exceptions:
    aiohttp.HttpBadRequest - On 404s
    VNDBOneResult - When you search for something but it instead redirects us to a direct content page
    VNDBNoResults - When nothing was found for that search
    VNDBBadStype - Raised when an incorrect search type is passed


async def get_novel(self, term, hide_nsfw=False):

if not term.isdigit() and not term.startswith('<STR_LIT:v>'):<EOL><INDENT>try:<EOL><INDENT>vnid = await self.search_vndb('<STR_LIT:v>', term)<EOL>vnid = vnid[<NUM_LIT:0>]['<STR_LIT:id>']<EOL><DEDENT>except VNDBOneResult as e:<EOL><INDENT>vnid = e.vnid<EOL><DEDENT><DEDENT>else:<EOL><INDENT>vnid = str(term)<EOL>if not vnid.startswith('<STR_LIT:v>'):<EOL><INDENT>vnid = '<STR_LIT:v>' + vnid<EOL><DEDENT><DEDENT>async with self.session.get(self.base_url + "<STR_LIT>".format(vnid), headers=self.headers) as response:<EOL><INDENT>if response.status == <NUM_LIT>:<EOL><INDENT>raise aiohttp.HttpBadRequest("<STR_LIT>".format(vnid))<EOL><DEDENT>text = await response.text()<EOL>soup = BeautifulSoup(text, '<STR_LIT>')<EOL>data = {'<STR_LIT>': {'<STR_LIT>': [], '<STR_LIT>': [], '<STR_LIT>': []}, '<STR_LIT>': None, '<STR_LIT>': None, '<STR_LIT>': [], '<STR_LIT>': [], '<STR_LIT>': {}, '<STR_LIT>': {}, '<STR_LIT:id>': vnid}<EOL>data['<STR_LIT>']['<STR_LIT>'] = soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>].h1.string<EOL>try:<EOL><INDENT>data['<STR_LIT>']['<STR_LIT>'] = soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>].string<EOL><DEDENT>except IndexError:<EOL><INDENT>data['<STR_LIT>']['<STR_LIT>'] = None<EOL><DEDENT>try:<EOL><INDENT>imgdiv = soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>]<EOL>if not (hide_nsfw and '<STR_LIT:class>' in imgdiv.p.attrs):<EOL><INDENT>data['<STR_LIT>'] = '<STR_LIT>' + imgdiv.img.get('<STR_LIT:src>')<EOL><DEDENT><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT>for item in soup.find_all('<STR_LIT>'):<EOL><INDENT>if '<STR_LIT:class>' in item.attrs or len(list(item.children)) == <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>if item.td.string == '<STR_LIT>':<EOL><INDENT>tlist = []<EOL>for alias in list(item.children)[<NUM_LIT:1>:]:<EOL><INDENT>tlist.append(alias.string)<EOL><DEDENT>data['<STR_LIT>']['<STR_LIT>'] = tlist<EOL><DEDENT>elif item.td.string == '<STR_LIT>':<EOL><INDENT>data['<STR_LIT>'] = list(item.children)[<NUM_LIT:1>].string<EOL><DEDENT>elif item.td.string == '<STR_LIT>':<EOL><INDENT>tl = []<EOL>for item in list(list(item.children)[<NUM_LIT:1>].children):<EOL><INDENT>if isinstance(item, NavigableString):<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT>' in item.attrs:<EOL><INDENT>tl.append(item.string)<EOL><DEDENT><DEDENT>data['<STR_LIT>'] = tl<EOL>del tl<EOL><DEDENT>elif item.td.string == '<STR_LIT>':<EOL><INDENT>tl = []<EOL>for item in list(list(item.children)[<NUM_LIT:1>].children):<EOL><INDENT>if isinstance(item, NavigableString):<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT>' in item.attrs:<EOL><INDENT>tl.append(item.string)<EOL><DEDENT><DEDENT>data['<STR_LIT>'] = tl<EOL><DEDENT><DEDENT>conttags = []<EOL>techtags = []<EOL>erotags = []<EOL>test = soup.find('<STR_LIT>', attrs={'<STR_LIT:id>': '<STR_LIT>'})<EOL>if test:<EOL><INDENT>for item in list(test.children):<EOL><INDENT>if isinstance(item, NavigableString):<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT:class>' not in item.attrs:<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT>' in "<STR_LIT:U+0020>".join(item.get('<STR_LIT:class>')):<EOL><INDENT>conttags.append(item.a.string)<EOL><DEDENT>if '<STR_LIT>' in "<STR_LIT:U+0020>".join(item.get('<STR_LIT:class>')):<EOL><INDENT>techtags.append(item.a.string)<EOL><DEDENT>if '<STR_LIT>' in "<STR_LIT:U+0020>".join(item.get('<STR_LIT:class>')):<EOL><INDENT>erotags.append(item.a.string)<EOL><DEDENT><DEDENT><DEDENT>data['<STR_LIT>']['<STR_LIT:content>'] = conttags if len(conttags) else None<EOL>data['<STR_LIT>']['<STR_LIT>'] = techtags if len(techtags) else None<EOL>data['<STR_LIT>']['<STR_LIT>'] = erotags if len(erotags) else None<EOL>del conttags<EOL>del techtags<EOL>del erotags<EOL>releases = []<EOL>cur_lang = None<EOL>for item in list(soup.find('<STR_LIT>', class_='<STR_LIT>').table.children):<EOL><INDENT>if isinstance(item, NavigableString):<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT:class>' in item.attrs:<EOL><INDENT>if cur_lang is None:<EOL><INDENT>cur_lang = item.td.abbr.get('<STR_LIT:title>')<EOL><DEDENT>else:<EOL><INDENT>data['<STR_LIT>'][cur_lang] = releases<EOL>releases = []<EOL>cur_lang = item.td.abbr.get('<STR_LIT:title>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>temp_rel = {'<STR_LIT:date>': <NUM_LIT:0>, '<STR_LIT>': <NUM_LIT:0>, '<STR_LIT>': <NUM_LIT:0>, '<STR_LIT:name>': <NUM_LIT:0>, '<STR_LIT:id>': <NUM_LIT:0>}<EOL>children = list(item.children)<EOL>temp_rel['<STR_LIT:date>'] = children[<NUM_LIT:0>].string<EOL>temp_rel['<STR_LIT>'] = children[<NUM_LIT:1>].string<EOL>temp_rel['<STR_LIT>'] = children[<NUM_LIT:2>].abbr.get('<STR_LIT:title>')<EOL>temp_rel['<STR_LIT:name>'] = children[<NUM_LIT:3>].a.string<EOL>temp_rel['<STR_LIT:id>'] = children[<NUM_LIT:3>].a.get('<STR_LIT>')[<NUM_LIT:1>:]<EOL>del children<EOL>releases.append(temp_rel)<EOL>del temp_rel<EOL><DEDENT><DEDENT>if len(releases) > <NUM_LIT:0> and cur_lang is not None:<EOL><INDENT>data['<STR_LIT>'][cur_lang] = releases<EOL><DEDENT>del releases<EOL>del cur_lang<EOL>desc = "<STR_LIT>"<EOL>for item in list(soup.find_all('<STR_LIT>', class_='<STR_LIT>')[<NUM_LIT:0>].children)[<NUM_LIT:1>].contents:<EOL><INDENT>if not isinstance(item, NavigableString):<EOL><INDENT>continue<EOL><DEDENT>if item.startswith('<STR_LIT:[>'):<EOL><INDENT>continue<EOL><DEDENT>if item.endswith('<STR_LIT:]>'):<EOL><INDENT>continue<EOL><DEDENT>desc += item.string + "<STR_LIT:\n>"<EOL><DEDENT>data['<STR_LIT:description>'] = desc<EOL>return data<EOL><DEDENT>

If term is an ID will return that specific ID. If it's a string, it will return the details of the first search result for that term.
Returned Dictionary Has the following structure:
Please note, if it says list or dict, it means the python types.
Indentation indicates level. So English is ['Titles']['English']

'Titles' - Contains all the titles found for the anime
    'English' - English title of the novel
    'Alt' - Alternative title (Usually the Japanese one, but other languages exist)
    'Aliases' - A list of str that define the aliases as given in VNDB.
'Img' - Link to the Image shown on VNDB for that Visual Novel
'Length' - Length given by VNDB
'Developers' - A list containing the Developers of the VN.
'Publishers' - A list containing the Publishers of the VN.
'Tags' - Contains 3 lists of different tag categories
    'Content' - List of tags that have to do with the story's content as defined by VNDB. Ex: Edo Era
    'Technology' - List of tags that have to do with the VN's technology. Ex: Protagonist with a Face (Wew Lad, 21st century)
    'Erotic' - List of tags that have to do with the VN's sexual content. Ex: Tentacles
'Releases' - A list of dictionaries. They have the following format.
    'Date' - Date VNDB lists for release
    'Ages' - Age group appropriate for as determined on VNDB
    'Platform' - Release Platform
    'Name' - The name for this particular Release
    'ID' - The id for this release, also doubles as the link if you append https://vndb.org/ to it
'Description' - Contains novel description text if there is any.
'ID' - The id for this novel, also doubles as the link if you append https://vndb.org/ to it

:param term: id or name to get details of.
:param hide_nsfw: bool if 'Img' should filter links flagged as NSFW or not. (no reason to be kwargs...yet)
:return dict: Dictionary with the parsed results of a novel


async def parse_search(self, stype, soup):

if stype == '<STR_LIT:v>':<EOL><INDENT>return await parse_vn_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:r>':<EOL><INDENT>return await parse_release_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:p>':<EOL><INDENT>return await parse_prod_staff_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:s>':<EOL><INDENT>return await parse_prod_staff_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:c>':<EOL><INDENT>return await parse_character_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:g>':<EOL><INDENT>return await parse_tag_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:i>':<EOL><INDENT>return await parse_tag_results(soup)<EOL><DEDENT>elif stype == '<STR_LIT:u>':<EOL><INDENT>return await parse_user_results(soup)<EOL><DEDENT>

This is our parsing dispatcher

:param stype: Search type category
:param soup: The beautifulsoup object that contains the parsed html


assert issubclass(method.im_class, NativeABIContract), method.im_class<EOL>m_abi = method.im_class._get_method_abi(method)<EOL>return zpad(encode_int(m_abi['<STR_LIT:id>']), <NUM_LIT:4>) + abi.encode_abi(m_abi['<STR_LIT>'], args)<EOL>


f.is_constant = True<EOL>return f<EOL>


def chain_nac_proxy(chain, sender, contract_address, value=<NUM_LIT:0>):

klass = registry[contract_address].im_self<EOL>assert issubclass(klass, NativeABIContract)<EOL>def mk_method(method):<EOL><INDENT>def m(s, *args):<EOL><INDENT>data = abi_encode_args(method, args)<EOL>block = chain.head_candidate<EOL>output = test_call(block, sender, contract_address, data)<EOL>if output is not None:<EOL><INDENT>return abi_decode_return_vals(method, output)<EOL><DEDENT><DEDENT>return m<EOL><DEDENT>class cproxy(object):<EOL><INDENT>pass<EOL><DEDENT>for m in klass._abi_methods():<EOL><INDENT>setattr(cproxy, m.__func__.func_name, mk_method(m))<EOL><DEDENT>return cproxy()<EOL>

create an object which acts as a proxy for the contract on the chain


assert isinstance(address, bytes) and len(address) == <NUM_LIT:20><EOL>assert self.is_instance_address(address)<EOL>nca = self.native_contract_address_prefix + address[-<NUM_LIT:4>:]<EOL>return self.native_contracts[nca]<EOL>

returns class._on_msg_unsafe, use x.im_self to get class


assert issubclass(contract, NativeContractBase)<EOL>assert len(contract.address) == <NUM_LIT:20><EOL>assert contract.address.startswith(self.native_contract_address_prefix)<EOL>if self.native_contracts.get(contract.address) == contract._on_msg:<EOL><INDENT>log.debug("<STR_LIT>", contract=contract, address=contract.address)<EOL>return<EOL><DEDENT>assert contract.address not in self.native_contracts, '<STR_LIT>'<EOL>self.native_contracts[contract.address] = contract._on_msg<EOL>log.debug("<STR_LIT>", contract=contract, address=contract.address)<EOL>


return <NUM_LIT:1>, self.gas, []<EOL>

method which gets called by default if no other method is found
note: this is no abi method and must make sense from self.msg_data


if self.__isfrozen and not hasattr(self, key):<EOL><INDENT>raise TypeError("<STR_LIT>" % self)<EOL><DEDENT>object.__setattr__(self, key, value)<EOL>


<EOL>self.project.get_service('<STR_LIT>').options['<STR_LIT>'] = ["<STR_LIT>"]<EOL>bootstrap = self.project.get_service('<STR_LIT>')<EOL>bootstrap.options['<STR_LIT>'] = [p.replace("<STR_LIT>", "<STR_LIT>")<EOL>for p in bootstrap.options['<STR_LIT>']]<EOL>for service in self.project.services:<EOL><INDENT>container_name = service.options.get('<STR_LIT>')<EOL>if container_name:<EOL><INDENT>service.options['<STR_LIT>'] = container_name.replace("<STR_LIT>",<EOL>PROJECT_NAME)<EOL><DEDENT><DEDENT>

Modify the services configurations to allow testing.


def initialize(ctx, a='<STR_LIT>', c='<STR_LIT:bool>', d='<STR_LIT>'):

Constructor (can a constructor return anything?)


result = []<EOL>for validator in validators:<EOL><INDENT>if len(validator) == <NUM_LIT>:<EOL><INDENT>validator = validator.decode('<STR_LIT>')<EOL><DEDENT>result.append(validator)<EOL><DEDENT>return result<EOL>

Consolidate (potentially hex-encoded) list of validators
    into list of binary address representations.


if data not in self.filter:<EOL><INDENT>self.filter.append(data)<EOL>if len(self.filter) > self.max_items:<EOL><INDENT>self.filter.pop(<NUM_LIT:0>)<EOL><DEDENT>return True<EOL><DEDENT>else:<EOL><INDENT>self.filter.append(self.filter.pop(<NUM_LIT:0>))<EOL>return False<EOL><DEDENT>


def add_transaction(self, tx, origin=None, force_broadcast=False):

self.consensus_manager.log(<EOL>'<STR_LIT>', blk=self.chain.head_candidate, lock=self.proposal_lock)<EOL>log.debug('<STR_LIT>', lock=self.proposal_lock)<EOL>block = self.proposal_lock.block<EOL>self.proposal_lock.acquire()<EOL>self.consensus_manager.log('<STR_LIT>', lock=self.proposal_lock)<EOL>assert not hasattr(self.chain.head_candidate, '<STR_LIT>')<EOL>success = super(ChainService, self).add_transaction(tx, origin, force_broadcast)<EOL>if self.proposal_lock.is_locked():  <EOL><INDENT>self.proposal_lock.release(if_block=block)<EOL><DEDENT>log.debug('<STR_LIT>', num_txs=self.chain.head_candidate.num_transactions())<EOL>return success<EOL>

Warning:
Locking proposal_lock may block incoming events which are necessary to unlock!
I.e. votes / blocks!
Take care!


def on_receive_transactions(self, proto, transactions):

log.debug('<STR_LIT>')<EOL>log.debug('<STR_LIT>', count=len(transactions), remote_id=proto)<EOL>def _add_txs():<EOL><INDENT>for tx in transactions:<EOL><INDENT>self.add_transaction(tx, origin=proto)<EOL><DEDENT><DEDENT>gevent.spawn(_add_txs)<EOL>


def transfer(ctx, _to='<STR_LIT:address>', _value='<STR_LIT>', returns=STATUS):

log.DEV('<STR_LIT>')<EOL>if ctx.accounts[ctx.msg_sender] >= _value:<EOL><INDENT>ctx.accounts[ctx.msg_sender] -= _value<EOL>ctx.accounts[_to] += _value<EOL>ctx.Transfer(ctx.msg_sender, _to, _value)<EOL>return OK<EOL><DEDENT>else:<EOL><INDENT>return INSUFFICIENTFUNDS<EOL><DEDENT>

Standardized Contract API:
        function transfer(address _to, uint256 _value) returns (bool _success)


def transferFrom(ctx, _from='<STR_LIT:address>', _to='<STR_LIT:address>', _value='<STR_LIT>', returns=STATUS):

auth = ctx.allowances[_from][ctx.msg_sender]<EOL>if ctx.accounts[_from] >= _value and auth >= _value:<EOL><INDENT>ctx.allowances[_from][ctx.msg_sender] -= _value<EOL>ctx.accounts[_from] -= _value<EOL>ctx.accounts[_to] += _value<EOL>ctx.Transfer(_from, _to, _value)<EOL>return OK<EOL><DEDENT>else:<EOL><INDENT>return INSUFFICIENTFUNDS<EOL><DEDENT>

Standardized Contract API:
        function transferFrom(address _from, address _to, uint256 _value) returns (bool success)


return ctx.supply<EOL>

Standardized Contract API:
        function totalSupply() constant returns (uint256 supply)


@nc.constant<EOL><INDENT>def balanceOf(ctx, _address='<STR_LIT:address>', returns='<STR_LIT>'):<DEDENT>

return ctx.accounts[_address]<EOL>

Standardized Contract API:
        function balanceOf(address _address) constant returns (uint256 balance)


def approve(ctx, _spender='<STR_LIT:address>', _value='<STR_LIT>', returns=STATUS):

ctx.allowances[ctx.msg_sender][_spender] += _value<EOL>ctx.Approval(ctx.msg_sender, _spender, _value)<EOL>return OK<EOL>

Standardized Contract API:
        function approve(address _spender, uint256 _value) returns (bool success)


@nc.constant<EOL><INDENT>def allowance(ctx, _spender='<STR_LIT:address>', returns='<STR_LIT>'):<DEDENT>

return ctx.allowances[ctx.msg_sender][_spender]<EOL>

Standardized Contract API:
        function allowance(address _owner, address _spender) constant returns (uint256 remaining)


def issue_funds(ctx, amount='<STR_LIT>', rtgs_hash='<STR_LIT>', returns=STATUS):

<EOL>ctx.accounts[ctx.msg_sender] += amount<EOL>ctx.issued_amounts[ctx.msg_sender] += amount<EOL>ctx.Issuance(ctx.msg_sender, rtgs_hash, amount)<EOL>return OK<EOL>

In the IOU fungible the supply is set by Issuer, who issue funds.


chain = app.services.chain<EOL>new_block_evt = gevent.event.Event()<EOL>def _on_new_block(app):<EOL><INDENT>log.DEV('<STR_LIT>')<EOL>new_block_evt.set()<EOL><DEDENT>chain.on_new_head_cbs.append(_on_new_block)<EOL>def wait_next_block():<EOL><INDENT>bn = chain.chain.head.number<EOL>chain.consensus_manager.log('<STR_LIT>', block=bn)<EOL>new_block_evt.wait(timeout)<EOL>new_block_evt.clear()<EOL>if chain.chain.head.number > bn:<EOL><INDENT>chain.consensus_manager.log('<STR_LIT>', block=chain.chain.head.number)<EOL><DEDENT>elif chain.chain.head.number == bn:<EOL><INDENT>chain.consensus_manager.log('<STR_LIT>', block=bn)<EOL><DEDENT><DEDENT>return wait_next_block<EOL>

Creates a `wait_next_block` function, that
    will wait `timeout` seconds (`None` = indefinitely)
    for a new block to appear.

    :param app: the app-instance the function should work for
    :param timeout: timeout in seconds


rs = list(self.rounds)<EOL>assert len(rs) < <NUM_LIT:2> or rs[<NUM_LIT:0>] > rs[<NUM_LIT:1>]  <EOL>for r in self.rounds:  <EOL><INDENT>if self.rounds[r].lock is not None:<EOL><INDENT>return self.rounds[r].lock<EOL><DEDENT><DEDENT>


for r in self.rounds:<EOL><INDENT>if isinstance(self.rounds[r].proposal, BlockProposal):<EOL><INDENT>assert isinstance(self.rounds[r].lock, Vote)<EOL>if self.rounds[r].proposal.blockhash == self.rounds[r].lock.blockhash:<EOL><INDENT>return self.rounds[r].proposal<EOL><DEDENT><DEDENT><DEDENT>


for r in self.rounds:<EOL><INDENT>ls = self.rounds[r].lockset<EOL>if ls.is_valid:<EOL><INDENT>return ls<EOL><DEDENT><DEDENT>return None<EOL>


if self.timeout_time is not None or self.proposal:<EOL><INDENT>return<EOL><DEDENT>now = self.cm.chainservice.now<EOL>round_timeout = ConsensusManager.round_timeout<EOL>round_timeout_factor = ConsensusManager.round_timeout_factor<EOL>delay = round_timeout * round_timeout_factor ** self.round<EOL>self.timeout_time = now + delay<EOL>return delay<EOL>

setup a timeout for waiting for a proposal


v = VoteBlock(<NUM_LIT:0>, <NUM_LIT:0>, genesis.hash)<EOL>v.sign(privkey)<EOL>ls = LockSet(num_eligible_votes=<NUM_LIT:1>)<EOL>ls.add(v)<EOL>assert ls.has_quorum<EOL>return ls<EOL>

in order to avoid a complicated bootstrapping, we define
the genesis_signing_lockset as a lockset with one vote by any validator.


if self.v:<EOL><INDENT>raise InvalidSignature("<STR_LIT>")<EOL><DEDENT>if privkey in (<NUM_LIT:0>, '<STR_LIT>', '<STR_LIT:\x00>' * <NUM_LIT:32>):<EOL><INDENT>raise InvalidSignature("<STR_LIT>")<EOL><DEDENT>rawhash = sha3(rlp.encode(self, self.__class__.exclude(['<STR_LIT:v>', '<STR_LIT:r>', '<STR_LIT:s>'])))<EOL>if len(privkey) == <NUM_LIT:64>:<EOL><INDENT>privkey = encode_privkey(privkey, '<STR_LIT>')<EOL><DEDENT>pk = PrivateKey(privkey, raw=True)<EOL>signature = pk.ecdsa_recoverable_serialize(pk.ecdsa_sign_recoverable(rawhash, raw=True))<EOL>signature = signature[<NUM_LIT:0>] + chr(signature[<NUM_LIT:1>])<EOL>self.v = ord(signature[<NUM_LIT:64>]) + <NUM_LIT><EOL>self.r = big_endian_to_int(signature[<NUM_LIT:0>:<NUM_LIT:32>])<EOL>self.s = big_endian_to_int(signature[<NUM_LIT:32>:<NUM_LIT:64>])<EOL>self._sender = None<EOL>return self<EOL>


if self.sender is None:<EOL><INDENT>raise MissingSignatureError()<EOL><DEDENT>class HashSerializable(rlp.Serializable):<EOL><INDENT>fields = [(field, sedes) for field, sedes in self.fields<EOL>if field not in ('<STR_LIT:v>', '<STR_LIT:r>', '<STR_LIT:s>')] + [('<STR_LIT>', binary)]<EOL>_sedes = None<EOL><DEDENT>return sha3(rlp.encode(self, HashSerializable))<EOL>


assert len(self), '<STR_LIT>'<EOL>h = set([(v.height, v.round) for v in self.votes])<EOL>assert len(h) == <NUM_LIT:1>, len(h)<EOL>return h.pop()<EOL>

compute (height,round)
        We might have multiple rounds before we see consensus for a certain height.
        If everything is good, round should always be 0.


assert self.is_valid<EOL>bhs = self.blockhashes()<EOL>if bhs and bhs[<NUM_LIT:0>][<NUM_LIT:1>] > <NUM_LIT:2> / <NUM_LIT> * self.num_eligible_votes:<EOL><INDENT>return bhs[<NUM_LIT:0>][<NUM_LIT:0>]<EOL><DEDENT>

we've seen +2/3 of all eligible votes voting for one block.
there is a quorum.


assert self.is_valid<EOL>bhs = self.blockhashes()<EOL>if not bhs or bhs[<NUM_LIT:0>][<NUM_LIT:1>] <= <NUM_LIT:1> / <NUM_LIT> * self.num_eligible_votes:<EOL><INDENT>assert not self.has_quorum_possible<EOL>return True<EOL><DEDENT>

less than 1/3 of the known votes are on the same block


if self.has_quorum:<EOL><INDENT>return<EOL><DEDENT>assert self.is_valid  <EOL>bhs = self.blockhashes()<EOL>if bhs and bhs[<NUM_LIT:0>][<NUM_LIT:1>] > <NUM_LIT:1> / <NUM_LIT> * self.num_eligible_votes:<EOL><INDENT>return bhs[<NUM_LIT:0>][<NUM_LIT:0>]<EOL><DEDENT>

we've seen +1/3 of all eligible votes voting for one block.
at least one vote was from a honest node.
we can assume that this block is agreeable.


if not self.is_valid:<EOL><INDENT>return True<EOL><DEDENT>test = (self.has_quorum, self.has_quorum_possible, self.has_noquorum)<EOL>assert <NUM_LIT:1> == len([x for x in test if x is not None])<EOL>return True<EOL>

either invalid or one of quorum, noquorum, quorumpossible


return Block(self.header, self.transaction_list, self.uncles, env=env, parent=parent)<EOL>

Convert the transient block to a :class:`ethereum.blocks.Block`


return sha3(rlp.encode(self.header))<EOL>

The binary block hash
        This is equivalent to ``header.hash``.


def __init__(self, height, round, block, signing_lockset,<EOL>round_lockset=None, v=<NUM_LIT:0>, r=<NUM_LIT:0>, s=<NUM_LIT:0>):

assert isinstance(block, (Block, TransientBlock))<EOL>assert isinstance(signing_lockset, LockSet)<EOL>assert round_lockset is None or isinstance(round_lockset, LockSet)<EOL>assert round >= <NUM_LIT:0><EOL>assert height > <NUM_LIT:0><EOL>if round > <NUM_LIT:0> and not round_lockset:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if round == <NUM_LIT:0> and round_lockset:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>self.height = height<EOL>self.round = round<EOL>self.block = block<EOL>self.signing_lockset = signing_lockset<EOL>self.round_lockset = round_lockset or LockSet(<NUM_LIT:0>)<EOL>super(BlockProposal, self).__init__(height, round, block, signing_lockset,<EOL>self.round_lockset, v, r, s)<EOL>if block.header.number != self.height:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if self.round_lockset and height != self.round_lockset.height:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if not (round > <NUM_LIT:0> or self.lockset.has_quorum):<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if not (round > <NUM_LIT:0> or self.lockset.height == block.header.number - <NUM_LIT:1>):<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if not (round == <NUM_LIT:0> or round == self.lockset.round + <NUM_LIT:1>):<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if not self.signing_lockset.has_quorum:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if not (self.signing_lockset.height == self.height - <NUM_LIT:1>):<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>if self.round_lockset and not round_lockset.has_noquorum:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>self.rawhash = sha3(rlp.encode(self, self.__class__.exclude(['<STR_LIT:v>', '<STR_LIT:r>', '<STR_LIT:s>'])))<EOL>if self.v:  <EOL><INDENT>assert self.sender<EOL><DEDENT>

if round == 0 the signing_lockset also proves,
that proposal is eligible and we need not round_lockset


def validate_votes(self, validators_H, validators_prevH):

assert self.sender<EOL>def check(lockset, validators):<EOL><INDENT>if not lockset.num_eligible_votes == len(validators):<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT>for v in lockset:<EOL><INDENT>if v.sender not in validators:<EOL><INDENT>raise InvalidProposalError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>if self.round_lockset:<EOL><INDENT>check(self.round_lockset, validators_H)<EOL><DEDENT>check(self.signing_lockset, validators_prevH)<EOL>return True<EOL>

set of validators may change between heights