def getCoverage(self, identifier=None, bbox=None, time=None, format = None,  subsets=None,crs=None, width=None, height=None, resx=None, resy=None, resz=None,parameter=None,method='<STR_LIT>',**kwargs):

if log.isEnabledFor(logging.DEBUG):<EOL><INDENT>log.debug('<STR_LIT>'%(identifier, bbox, time, format, crs, width, height, resx, resy, resz, parameter, method, str(kwargs)))<EOL><DEDENT>try:<EOL><INDENT>base_url = next((m.get('<STR_LIT:url>') for m in self.getOperationByName('<STR_LIT>').methods if m.get('<STR_LIT:type>').lower() == method.lower()))<EOL><DEDENT>except StopIteration:<EOL><INDENT>base_url = self.url<EOL><DEDENT>if log.isEnabledFor(logging.DEBUG):<EOL><INDENT>log.debug('<STR_LIT>'%base_url)<EOL><DEDENT>request = {'<STR_LIT:version>': self.version, '<STR_LIT>': '<STR_LIT>', '<STR_LIT>':'<STR_LIT>'}<EOL>assert len(identifier) > <NUM_LIT:0><EOL>request['<STR_LIT>']=identifier[<NUM_LIT:0>]<EOL>if crs:<EOL><INDENT>request['<STR_LIT>']=crs<EOL><DEDENT>request['<STR_LIT>']=format<EOL>if width:<EOL><INDENT>request['<STR_LIT:width>']=width<EOL><DEDENT>if height:<EOL><INDENT>request['<STR_LIT>']=height<EOL><DEDENT>if kwargs:<EOL><INDENT>for kw in kwargs:<EOL><INDENT>request[kw]=kwargs[kw]<EOL><DEDENT><DEDENT>data = urlencode(request)<EOL>if subsets:<EOL><INDENT>for subset in subsets:<EOL><INDENT>if len(subset) > <NUM_LIT:2>:<EOL><INDENT>if not self.is_number(subset[<NUM_LIT:1>]):<EOL><INDENT>data = data + "<STR_LIT:&>"+ urlencode({"<STR_LIT>":subset[<NUM_LIT:0>]+'<STR_LIT>'+self.__makeString(subset[<NUM_LIT:1>])+'<STR_LIT>'+self.__makeString(subset[<NUM_LIT:2>])+'<STR_LIT>'})<EOL><DEDENT>else:<EOL><INDENT>data = data + "<STR_LIT:&>"+ urlencode({"<STR_LIT>":subset[<NUM_LIT:0>]+'<STR_LIT:(>'+self.__makeString(subset[<NUM_LIT:1>])+'<STR_LIT:U+002C>'+self.__makeString(subset[<NUM_LIT:2>])+'<STR_LIT:)>'})<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not self.is_number(subset[<NUM_LIT:1>]):<EOL><INDENT>data = data + "<STR_LIT:&>"+ urlencode({"<STR_LIT>":subset[<NUM_LIT:0>]+'<STR_LIT>'+self.__makeString(subset[<NUM_LIT:1>])+'<STR_LIT>'})<EOL><DEDENT>else:<EOL><INDENT>data = data + "<STR_LIT:&>"+ urlencode({"<STR_LIT>":subset[<NUM_LIT:0>]+'<STR_LIT:(>'+self.__makeString(subset[<NUM_LIT:1>])+'<STR_LIT:)>'})<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if log.isEnabledFor(logging.DEBUG):<EOL><INDENT>log.debug('<STR_LIT>'%data)<EOL><DEDENT>u=openURL(base_url, data, method, self.cookies)<EOL>return u<EOL>

Request and return a coverage from the WCS as a file-like object
        note: additional **kwargs helps with multi-version implementation
        core keyword arguments should be supported cross version
        example:
        cvg=wcs.getCoverage(identifier=['TuMYrRQ4'], timeSequence=['2792-06-01T00:00:00.0'], bbox=(-112,36,-106,41),format='cf-netcdf')

        is equivalent to:
        http://myhost/mywcs?SERVICE=WCS&REQUEST=GetCoverage&IDENTIFIER=TuMYrRQ4&VERSION=1.1.0&BOUNDINGBOX=-180,-90,180,90&TIME=2792-06-01T00:00:00.0&FORMAT=cf-netcdf

        example 2.0.1 URL
        http://earthserver.pml.ac.uk/rasdaman/ows?&SERVICE=WCS&VERSION=2.0.1&REQUEST=GetCoverage
        &COVERAGEID=V2_monthly_CCI_chlor_a_insitu_test&SUBSET=Lat(40,50)&SUBSET=Long(-10,0)&SUBSET=ansi(144883,145000)&FORMAT=application/netcdf

        cvg=wcs.getCoverage(identifier=['myID'], format='application/netcdf', subsets=[('axisName',min,max),('axisName',min,max),('axisName',min,max)])


def getCoverage(self, identifier=None, bbox=None, time=None, format = None,  crs=None, width=None, height=None, resx=None, resy=None, resz=None,parameter=None,method='<STR_LIT>',**kwargs):

if log.isEnabledFor(logging.DEBUG):<EOL><INDENT>log.debug('<STR_LIT>'%(identifier, bbox, time, format, crs, width, height, resx, resy, resz, parameter, method, str(kwargs)))<EOL><DEDENT>try:<EOL><INDENT>base_url = next((m.get('<STR_LIT:url>') for m in self.getOperationByName('<STR_LIT>').methods if m.get('<STR_LIT:type>').lower() == method.lower()))<EOL><DEDENT>except StopIteration:<EOL><INDENT>base_url = self.url<EOL><DEDENT>if log.isEnabledFor(logging.DEBUG):<EOL><INDENT>log.debug('<STR_LIT>'%base_url)<EOL><DEDENT>request = {'<STR_LIT:version>': self.version, '<STR_LIT>': '<STR_LIT>', '<STR_LIT>':'<STR_LIT>'}<EOL>assert len(identifier) > <NUM_LIT:0><EOL>request['<STR_LIT>']=identifier<EOL>if bbox:<EOL><INDENT>request['<STR_LIT>']='<STR_LIT:U+002C>'.join([self.__makeString(x) for x in bbox])<EOL><DEDENT>else:<EOL><INDENT>request['<STR_LIT>']=None<EOL><DEDENT>if time:<EOL><INDENT>request['<STR_LIT:time>']='<STR_LIT:U+002C>'.join(time)<EOL><DEDENT>if crs:<EOL><INDENT>request['<STR_LIT>']=crs<EOL><DEDENT>request['<STR_LIT>']=format<EOL>if width:<EOL><INDENT>request['<STR_LIT:width>']=width<EOL><DEDENT>if height:<EOL><INDENT>request['<STR_LIT>']=height<EOL><DEDENT>if resx:<EOL><INDENT>request['<STR_LIT>']=resx<EOL><DEDENT>if resy:<EOL><INDENT>request['<STR_LIT>']=resy<EOL><DEDENT>if resz:<EOL><INDENT>request['<STR_LIT>']=resz<EOL><DEDENT>if kwargs:<EOL><INDENT>for kw in kwargs:<EOL><INDENT>request[kw]=kwargs[kw]<EOL><DEDENT><DEDENT>data = urlencode(request)<EOL>if log.isEnabledFor(logging.DEBUG):<EOL><INDENT>log.debug('<STR_LIT>'%data)<EOL><DEDENT>u=openURL(base_url, data, method, self.cookies)<EOL>return u<EOL>

Request and return a coverage from the WCS as a file-like object
        note: additional **kwargs helps with multi-version implementation
        core keyword arguments should be supported cross version
        example:
        cvg=wcs.getCoverage(identifier=['TuMYrRQ4'], timeSequence=['2792-06-01T00:00:00.0'], bbox=(-112,36,-106,41),format='cf-netcdf')

        is equivalent to:
        http://myhost/mywcs?SERVICE=WCS&REQUEST=GetCoverage&IDENTIFIER=TuMYrRQ4&VERSION=1.1.0&BOUNDINGBOX=-180,-90,180,90&TIME=2792-06-01T00:00:00.0&FORMAT=cf-netcdf


<EOL>self._elem=elem<EOL>self._service=service<EOL>self.id=elem.find(ns('<STR_LIT:name>')).text<EOL>self.title = testXMLValue(elem.find(ns('<STR_LIT:label>')))<EOL>self.abstract= testXMLValue(elem.find(ns('<STR_LIT:description>')))<EOL>self.keywords = [f.text for f in elem.findall(ns('<STR_LIT>')+'<STR_LIT:/>'+ns('<STR_LIT>'))]        <EOL>self.boundingBox=None <EOL>self.boundingBoxWGS84 = None        <EOL>b = elem.find(ns('<STR_LIT>')) <EOL>if b is not None:<EOL><INDENT>gmlpositions=b.findall('<STR_LIT>')<EOL>lc=gmlpositions[<NUM_LIT:0>].text<EOL>uc=gmlpositions[<NUM_LIT:1>].text<EOL>self.boundingBoxWGS84 = (<EOL>float(lc.split()[<NUM_LIT:0>]),float(lc.split()[<NUM_LIT:1>]),<EOL>float(uc.split()[<NUM_LIT:0>]), float(uc.split()[<NUM_LIT:1>]),<EOL>)<EOL><DEDENT>self.styles=None<EOL>self.crsOptions=None<EOL>self.defaulttimeposition=None<EOL>

Initialize. service is required so that describeCoverage requests may be made


if name in self.__getattribute__('<STR_LIT>').keys():<EOL><INDENT>return self.__getattribute__('<STR_LIT>')[name]<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % name)<EOL><DEDENT>

check contents dictionary to allow dict like access to service layers


self.name = elem.tag.split('<STR_LIT:}>')[<NUM_LIT:1>]          <EOL>self.methods = []<EOL>for resource in elem.findall(ns('<STR_LIT>')+ns('<STR_LIT>')+ns('<STR_LIT>')+ns('<STR_LIT>')):<EOL><INDENT>url = resource.attrib['<STR_LIT>']<EOL>self.methods.append({'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT:url>': url})<EOL><DEDENT>for resource in elem.findall(ns('<STR_LIT>')+ns('<STR_LIT>')+ns('<STR_LIT>')+ns('<STR_LIT>')):<EOL><INDENT>url = resource.attrib['<STR_LIT>']<EOL>self.methods.append({'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT:url>': url})<EOL><DEDENT>


if not isinstance(st, bytes):<EOL><INDENT>raise ValueError("<STR_LIT>" % type(st))<EOL><DEDENT>return etree.fromstring(st)<EOL>

Parse a SOS capabilities document, returning an elementtree instance

st should be an XML capabilities document


qs = []<EOL>if service_url.find('<STR_LIT:?>') != -<NUM_LIT:1>:<EOL><INDENT>qs = cgi.parse_qsl(service_url.split('<STR_LIT:?>')[<NUM_LIT:1>])<EOL><DEDENT>params = [x[<NUM_LIT:0>] for x in qs]<EOL>if '<STR_LIT>' not in params:<EOL><INDENT>qs.append(('<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>if '<STR_LIT>' not in params:<EOL><INDENT>qs.append(('<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>if '<STR_LIT>' not in params:<EOL><INDENT>qs.append(('<STR_LIT>', self.version))<EOL><DEDENT>urlqs = urlencode(tuple(qs))<EOL>return service_url.split('<STR_LIT:?>')[<NUM_LIT:0>] + '<STR_LIT:?>' + urlqs<EOL>


pass<EOL>

Parse metadata elements relating to timeseries:
            TS: baseTime, spacing, commentBlock, parameter
            MTS: startAnchor, endAnchor, cumulative, accAnchor/Length, maxGap


if self.result is not None:<EOL><INDENT>result = self.result.find(nspv(<EOL>"<STR_LIT>"))<EOL>self.result = MeasurementTimeseries(result)<EOL><DEDENT>

Parse the result element of the observation type


self.quality = testXMLAttribute(element.find(nspv(<EOL>"<STR_LIT>")), nspv("<STR_LIT>"))<EOL>self.nilReason = testXMLAttribute(element.find(nspv(<EOL>"<STR_LIT>")), nspv("<STR_LIT>"))<EOL>self.comment = testXMLValue(element.find(nspv(<EOL>"<STR_LIT>")))<EOL>self.qualifier = testXMLAttribute(element.find(nspv(<EOL>"<STR_LIT>")), nspv("<STR_LIT>"))<EOL>self.processing = testXMLValue(element.find(nspv(<EOL>"<STR_LIT>")), nspv("<STR_LIT>"))<EOL>self.source = testXMLValue(element.find(nspv(<EOL>"<STR_LIT>")), nspv("<STR_LIT>"))<EOL>

Base time-value pair metadata. Still to do:
            - relatedObservation


print("<STR_LIT>")<EOL>wps = WebProcessingService('<STR_LIT>', verbose=verbose)<EOL>processid = '<STR_LIT>'<EOL>textdoc = ComplexDataInput("<STR_LIT>")   <EOL>inputs = [("<STR_LIT:text>", textdoc)]<EOL>outputs = [("<STR_LIT>",True,'<STR_LIT>')]<EOL>execution = wps.execute(processid, inputs, output=outputs)<EOL>monitorExecution(execution)<EOL>print('<STR_LIT>', execution.percentCompleted)<EOL>print('<STR_LIT>', execution.statusMessage)<EOL>for output in execution.processOutputs:<EOL><INDENT>print('<STR_LIT>' % (output.identifier, output.dataType, output.data, output.reference))<EOL><DEDENT>

use ComplexDataInput with a direct content


self.settings = self.read()<EOL>


if key.endswith("<STR_LIT:.>"):<EOL><INDENT>key = key[:-<NUM_LIT:1>]<EOL><DEDENT>path = key.split("<STR_LIT:.>")<EOL>curr = self.settings<EOL>for p in path[:-<NUM_LIT:1>]:<EOL><INDENT>if p not in curr:<EOL><INDENT>raise ConfigurationError("<STR_LIT>".format(key, p))<EOL><DEDENT>curr = curr[p]<EOL><DEDENT>if not isinstance(curr, dict):<EOL><INDENT>raise ConfigurationError("<STR_LIT>".format(path[-<NUM_LIT:1>], key))<EOL><DEDENT>if path[-<NUM_LIT:1>] not in curr:<EOL><INDENT>raise ConfigurationError("<STR_LIT>".format(key, path[-<NUM_LIT:1>]))<EOL><DEDENT>del curr[path[-<NUM_LIT:1>]]<EOL>

Remove a value at the given key -- and any nested values --
from the configuration.
*Note*: In order to write changes to the file, ensure that
:meth:`~giraffez.config.Config.write` is called prior to exit.

:param str key: A path to the value destination, with nested levels joined by '.'
:raises `giraffez.errors.ConfigurationError`: if the key specifies an invalid path, or does not exist


iv = "<STR_LIT>".format(os.urandom(<NUM_LIT:32>), time.time())<EOL>new_key = generate_key(ensure_bytes(iv))<EOL>with open(path, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write(base64.b64encode(new_key))<EOL><DEDENT>os.chmod(path, <NUM_LIT>)<EOL>

Creates a new encryption key in the path provided and sets the file
permissions.  Setting the file permissions currently does not work
on Windows platforms because of the differences in how file
permissions are read and modified.


if '<STR_LIT:b>' not in writer.mode:<EOL><INDENT>raise GiraffeError("<STR_LIT>")<EOL><DEDENT>writer.write(GIRAFFE_MAGIC)<EOL>writer.write(self.columns.serialize())<EOL>i = <NUM_LIT:0><EOL>for n, chunk in enumerate(self._fetchall(ROW_ENCODING_RAW), <NUM_LIT:1>):<EOL><INDENT>writer.write(chunk)<EOL>yield TeradataEncoder.count(chunk)<EOL><DEDENT>

Writes export archive files in the Giraffez archive format.
This takes a `giraffez.io.Writer` and writes archive chunks to
file until all rows for a given statement have been exhausted.

.. code-block:: python

    with giraffez.BulkExport("database.table_name") as export:
        with giraffez.Writer("database.table_name.tar.gz", 'wb', use_gzip=True) as out:
            for n in export.to_archive(out):
                print("Rows: {}".format(n))

:param `giraffez.io.Writer` writer: A writer handling the archive output

:rtype: iterator (yields ``int``)


return self._fetchall(ROW_ENCODING_DICT)<EOL>

Sets the current encoder output to Python `dict` and returns
a row iterator.

:rtype: iterator (yields ``dict``)


column_list = cls()<EOL>while data:<EOL><INDENT>tup, data = data[:<NUM_LIT:10>], data[<NUM_LIT:10>:]<EOL>column_type, length, prec, scale, title_len = struct.unpack("<STR_LIT>", tup)<EOL>title, data = data[:title_len], data[title_len:]<EOL>try:<EOL><INDENT>column_list.append((title, column_type, length, prec, scale))<EOL><DEDENT>except GiraffeTypeError as error:<EOL><INDENT>raise GiraffeEncodeError(error)<EOL><DEDENT><DEDENT>return column_list<EOL>

Deserializes giraffez Archive header. See
:meth:`~giraffez.types.Columns.serialize` for more information.

:param str data: data in giraffez Archive format, to be deserialized
:return: :class:`~giraffez.types.Columns` object decoded from data


_names = []<EOL>if names:<EOL><INDENT>for name in names:<EOL><INDENT>_safe_name = safe_name(name)<EOL>if _safe_name not in self._column_map:<EOL><INDENT>raise GiraffeTypeError("<STR_LIT>".format(name))<EOL><DEDENT>if _safe_name in _names:<EOL><INDENT>continue<EOL><DEDENT>_names.append(_safe_name)<EOL><DEDENT><DEDENT>self._filtered_columns = _names<EOL>

Set the names of columns to be used when iterating through the list,
retrieving names, etc.

:param list names: A list of names to be used, or :code:`None` for all


n = <NUM_LIT:0><EOL>for row in self:<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Exhausts the current connection by iterating over all rows and
returning the total.

.. code-block:: python

    with giraffez.Cmd() as cmd:
        results = cmd.execute("select * from dbc.dbcinfo")
        print(results.readall())


from ._teradata import register_graceful_shutdown_signal as _register<EOL>_register()<EOL>

Registers graceful shutdown handler using C signals. The first
SIGINT will attempt to close the Teradata connections before
exiting, and the second will shutdown whether the connections
have been closed or not.


return self.mload.checkpoint()<EOL>

Execute a checkpoint while loading rows. Called automatically
when loading from a file. Updates the exit code of the driver to
reflect errors.


return self._columns<EOL>

The list of columns in use.

:getter: Return the list of columns in use.
:setter: Set the columns to be loaded into, as well as their order. If
    loading from a file, these will be determined from the file header.
    Not necessary if you are loading into all columns, in the original
    order. The value must be a :code:`list` of names in the order that
    the fields of data will be presented in each row.

    Raises :class:`~giraffez.errors.GiraffeError` if :code:`field_names`
    is not a :code:`list`.

    Raises :class:`~giraffez.errors.GiraffeError` if the target table
    has not been set.
:type: :class:`~giraffez.types.Columns`


if not self.initiated:<EOL><INDENT>self._initiate()<EOL><DEDENT>try:<EOL><INDENT>row_status = self.mload.put_row(self.preprocessor(items))<EOL>self.applied_count += <NUM_LIT:1><EOL><DEDENT>except (TeradataPTError, EncoderError) as error:<EOL><INDENT>self.error_count += <NUM_LIT:1><EOL>if panic:<EOL><INDENT>raise error<EOL><DEDENT>log.info("<STR_LIT>", error)<EOL><DEDENT>

Load a single row into the target table.

:param list items: A list of values in the row corresponding to the
    fields specified by :code:`self.columns`
:param bool panic: If :code:`True`, when an error is encountered it will be
    raised. Otherwise, the error will be logged and :code:`self.error_count`
    is incremented.
:raises `giraffez.errors.GiraffeEncodeError`: if :code:`panic` is :code:`True` and there
    are format errors in the row values.
:raises `giraffez.errors.GiraffeError`: if table name is not set.
:raises `giraffez.TeradataPTError`: if there is a problem
    connecting to Teradata.


if platform.system() == '<STR_LIT>':<EOL><INDENT>if is_64bit():<EOL><INDENT>return latest_teradata_version("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return latest_teradata_version("<STR_LIT>")<EOL><DEDENT><DEDENT>elif platform.system() == '<STR_LIT>':<EOL><INDENT>return latest_teradata_version("<STR_LIT>")<EOL><DEDENT>elif platform.system() == '<STR_LIT>':<EOL><INDENT>return latest_teradata_version("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return latest_teradata_version("<STR_LIT>")<EOL><DEDENT>

Attempts to find the Teradata install directory with the defaults
for a given platform.  Should always return `None` when the defaults
are not present and the TERADATA_HOME environment variable wasn't
explicitly set to the correct install location.


import distutils.ccompiler<EOL>def _fix_compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=<NUM_LIT:0>,<EOL>extra_preargs=None, extra_postargs=None, depends=None):<EOL><INDENT>for flag in remove_flags:<EOL><INDENT>if flag in self.compiler_so:<EOL><INDENT>self.compiler_so.remove(flag)<EOL><DEDENT><DEDENT>macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros,<EOL>include_dirs, sources, depends, extra_postargs)<EOL>cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)<EOL>for obj in objects:<EOL><INDENT>try:<EOL><INDENT>src, ext = build[obj]<EOL><DEDENT>except KeyError:<EOL><INDENT>continue<EOL><DEDENT>self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)<EOL><DEDENT>return objects<EOL><DEDENT>distutils.ccompiler.CCompiler.compile = _fix_compile<EOL>

Monkey-patch compiler to allow for removal of default compiler flags.


searcher = Searcher(source="<STR_LIT>")<EOL>searcher.go()<EOL>


try:<EOL><INDENT>completed = subprocess.run(<EOL>command,<EOL>check=True,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE<EOL>)<EOL><DEDENT>except subprocess.CalledProcessError as err:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>print(completed.stdout.decode('<STR_LIT:utf-8>') + str("<STR_LIT::>") + completed.stderr.decode("<STR_LIT:utf-8>"))<EOL>return completed.stdout.decode('<STR_LIT:utf-8>')  + completed.stderr.decode("<STR_LIT:utf-8>")<EOL><DEDENT>

Execute shell command and return stdout txt
:param command:
:return:


def connection_remote_closed(self, connection, pn_condition):

assert len(self.receivers) == <NUM_LIT:0><EOL>self.connection.close()<EOL>

All senders have finished and closed, test over.


super(CyrusTest, self).setup()<EOL>if not CyrusTest.CONF_DIR:<EOL><INDENT>raise common.Skipped("<STR_LIT>")<EOL><DEDENT>self.container1 = pyngus.Container("<STR_LIT>")<EOL>self.container2 = pyngus.Container("<STR_LIT>")<EOL>

Create a simple SASL configuration. This assumes saslpasswd2 is in
        the OS path, otherwise the test will be skipped.


rl_handler = common.ReceiverCallback()<EOL>receiver = self.conn2.create_receiver("<STR_LIT>", "<STR_LIT:src>", rl_handler)<EOL>receiver.user_context = rl_handler<EOL>receiver.open()<EOL>self.process_connections()<EOL>assert self.conn1_handler.sender_requested_ct == <NUM_LIT:1><EOL>args = self.conn1_handler.sender_requested_args[<NUM_LIT:0>]<EOL>sl_handler = common.SenderCallback()<EOL>sender = self.conn1.accept_sender(args.link_handle,<EOL>event_handler=sl_handler)<EOL>sender.user_context = sl_handler<EOL>sender.open()<EOL>self.process_connections()<EOL>assert sender.active and sl_handler.active_ct > <NUM_LIT:0><EOL>assert receiver.active and rl_handler.active_ct > <NUM_LIT:0><EOL>return (sender, receiver)<EOL>


try:<EOL><INDENT>pyngus.read_socket_input(self.connection, self.socket)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOG.error("<STR_LIT>", str(e))<EOL>self.connection.close_input()<EOL>self.connection.close()<EOL><DEDENT>self.connection.process(time.time())<EOL>


def __init__(self, container, socket_, name, properties):

self.socket = socket_<EOL>self.connection = container.create_connection(name,<EOL>self,  <EOL>properties)<EOL>self.connection.user_context = self<EOL>self.connection.open()<EOL>self.sender_links = set()<EOL>self.receiver_links = set()<EOL>


def server_socket(host, port, backlog=<NUM_LIT:10>):

addr = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)<EOL>if not addr:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (host, str(port)))<EOL><DEDENT>my_socket = socket.socket(addr[<NUM_LIT:0>][<NUM_LIT:0>], addr[<NUM_LIT:0>][<NUM_LIT:1>], addr[<NUM_LIT:0>][<NUM_LIT:2>])<EOL>my_socket.setblocking(<NUM_LIT:0>)  <EOL>try:<EOL><INDENT>my_socket.bind(addr[<NUM_LIT:0>][<NUM_LIT:4>])<EOL>my_socket.listen(backlog)<EOL><DEDENT>except socket.error as e:<EOL><INDENT>if e.errno != errno.EINPROGRESS:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return my_socket<EOL>

Create a TCP listening socket for a server.


def connect_socket(host, port, blocking=True):

addr = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)<EOL>if not addr:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (host, str(port)))<EOL><DEDENT>my_socket = socket.socket(addr[<NUM_LIT:0>][<NUM_LIT:0>], addr[<NUM_LIT:0>][<NUM_LIT:1>], addr[<NUM_LIT:0>][<NUM_LIT:2>])<EOL>if not blocking:<EOL><INDENT>my_socket.setblocking(<NUM_LIT:0>)<EOL><DEDENT>try:<EOL><INDENT>my_socket.connect(addr[<NUM_LIT:0>][<NUM_LIT:4>])<EOL><DEDENT>except socket.error as e:<EOL><INDENT>if e.errno != errno.EINPROGRESS:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return my_socket<EOL>


def sender_failed(self, sender_link, error):

LOG.debug("<STR_LIT>", error)<EOL>sender_link.close()<EOL>


LOG.debug("<STR_LIT>", pn_condition)<EOL>connection.close()<EOL>

Peer has closed its end of the connection.


LOG.error("<STR_LIT>",<EOL>self._name, error)<EOL>


The remote has closed its end of the endpoint.


raise NotImplementedError("<STR_LIT>")<EOL>


LOG.debug("<STR_LIT>")<EOL>if self._handler:<EOL><INDENT>with self._callback_lock:<EOL><INDENT>self._handler.connection_active(self)<EOL><DEDENT><DEDENT>

Both ends of the Endpoint have become active.


def create_sender(self, source_address, target_address=None,<EOL>event_handler=None, name=None, properties=None):

ident = name or str(source_address)<EOL>if ident in self._sender_links:<EOL><INDENT>raise KeyError("<STR_LIT>" % ident)<EOL><DEDENT>session = _SessionProxy("<STR_LIT>" % ident, self)<EOL>session.open()<EOL>sl = session.new_sender(ident)<EOL>sl.configure(target_address, source_address, event_handler, properties)<EOL>self._sender_links[ident] = sl<EOL>return sl<EOL>


def wrap(self, *args, **kws):<EOL><INDENT>if self._callback_lock and self._callback_lock.in_callback:<EOL><INDENT>m = "<STR_LIT>" % func<EOL>raise RuntimeError(m)<EOL><DEDENT>return func(self, *args, **kws)<EOL><DEDENT>return wrap<EOL>

Decorator that prevents callbacks from calling into methods that are
        not reentrant


def reject_sender(self, link_handle, pn_condition=None):

link = self._sender_links.get(link_handle)<EOL>if not link:<EOL><INDENT>raise Exception("<STR_LIT>" % link_handle)<EOL><DEDENT>link.reject(pn_condition)<EOL>link.destroy()<EOL>

Rejects the SenderLink, and destroys the handle.


def sasl_done(self, connection, pn_sasl, result):


def create_receiver(self, target_address, source_address=None,<EOL>event_handler=None, name=None, properties=None):

ident = name or str(target_address)<EOL>if ident in self._receiver_links:<EOL><INDENT>raise KeyError("<STR_LIT>" % ident)<EOL><DEDENT>session = _SessionProxy("<STR_LIT>" % ident, self)<EOL>session.open()<EOL>rl = session.new_receiver(ident)<EOL>rl.configure(target_address, source_address, event_handler, properties)<EOL>self._receiver_links[ident] = rl<EOL>return rl<EOL>

Factory method for creating Receive links.


return self._pn_connection.remote_container<EOL>

Return the name of the remote container. Should be present once the
        connection is active.


def sasl_step(self, connection, pn_sasl):


return self._next_deadline<EOL>

Must invoke process() on or before this timestamp.


LOG.debug("<STR_LIT>", self._name)<EOL>


pn_link = self._pn_session.receiver(name)<EOL>return self.request_receiver(pn_link)<EOL>


self._links.discard(link)<EOL>if not self._links:<EOL><INDENT>LOG.debug("<STR_LIT>")<EOL>self._pn_session.close()<EOL>self._pn_session.free()<EOL>self._pn_session = None<EOL>self._connection = None<EOL><DEDENT>


<EOL>if self._pn_link.is_sender:<EOL><INDENT>return self._pn_link.source.address<EOL><DEDENT>else:<EOL><INDENT>return self._pn_link.remote_source.address<EOL><DEDENT>

Return the authorative source of the link.


def configure(self, target_address, source_address, handler, properties):

self._handler = handler<EOL>self._properties = properties<EOL>dynamic_props = None<EOL>if properties:<EOL><INDENT>dynamic_props = properties.get("<STR_LIT>")<EOL>mode = _dist_modes.get(properties.get("<STR_LIT>"))<EOL>if mode is not None:<EOL><INDENT>self._pn_link.source.distribution_mode = mode<EOL><DEDENT>mode = _snd_settle_modes.get(properties.get("<STR_LIT>"))<EOL>if mode is not None:<EOL><INDENT>self._pn_link.snd_settle_mode = mode<EOL><DEDENT>mode = _rcv_settle_modes.get(properties.get("<STR_LIT>"))<EOL>if mode is not None:<EOL><INDENT>self._pn_link.rcv_settle_mode = mode<EOL><DEDENT><DEDENT>if target_address is None:<EOL><INDENT>if not self._pn_link.is_sender:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self._pn_link.target.dynamic = True<EOL>if dynamic_props:<EOL><INDENT>self._pn_link.target.properties.clear()<EOL>self._pn_link.target.properties.put_dict(dynamic_props)<EOL><DEDENT><DEDENT>elif target_address:<EOL><INDENT>self._pn_link.target.address = target_address<EOL><DEDENT>if source_address is None:<EOL><INDENT>if not self._pn_link.is_receiver:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self._pn_link.source.dynamic = True<EOL>if dynamic_props:<EOL><INDENT>self._pn_link.source.properties.clear()<EOL>self._pn_link.source.properties.put_dict(dynamic_props)<EOL><DEDENT><DEDENT>elif source_address:<EOL><INDENT>self._pn_link.source.address = source_address<EOL><DEDENT>


def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,<EOL>to_dir=os.curdir, delay=<NUM_LIT:15>):

<EOL>to_dir = os.path.abspath(to_dir)<EOL>try:<EOL><INDENT>from urllib.request import urlopen<EOL><DEDENT>except ImportError:<EOL><INDENT>from urllib2 import urlopen<EOL><DEDENT>tgz_name = "<STR_LIT>" % version<EOL>url = download_base + tgz_name<EOL>saveto = os.path.join(to_dir, tgz_name)<EOL>src = dst = None<EOL>if not os.path.exists(saveto):  <EOL><INDENT>try:<EOL><INDENT>log.warn("<STR_LIT>", url)<EOL>src = urlopen(url)<EOL>data = src.read()<EOL>dst = open(saveto, "<STR_LIT:wb>")<EOL>dst.write(data)<EOL><DEDENT>finally:<EOL><INDENT>if src:<EOL><INDENT>src.close()<EOL><DEDENT>if dst:<EOL><INDENT>dst.close()<EOL><DEDENT><DEDENT><DEDENT>return os.path.realpath(saveto)<EOL>

Download distribute from a specified location and return its filename

    `version` should be a valid distribute version number that is available
    as an egg for download under the `download_base` URL (which should end
    with a '/'). `to_dir` is the directory where the egg will be downloaded.
    `delay` is the number of seconds to pause before an actual download
    attempt.


tarball = download_setuptools()<EOL>_install(tarball)<EOL>

Install or upgrade setuptools and EasyInstall


def _extractall(self, path="<STR_LIT:.>", members=None):

import copy<EOL>import operator<EOL>from tarfile import ExtractError<EOL>directories = []<EOL>if members is None:<EOL><INDENT>members = self<EOL><DEDENT>for tarinfo in members:<EOL><INDENT>if tarinfo.isdir():<EOL><INDENT>directories.append(tarinfo)<EOL>tarinfo = copy.copy(tarinfo)<EOL>tarinfo.mode = <NUM_LIT> <EOL><DEDENT>self.extract(tarinfo, path)<EOL><DEDENT>if sys.version_info < (<NUM_LIT:2>, <NUM_LIT:4>):<EOL><INDENT>def sorter(dir1, dir2):<EOL><INDENT>return cmp(dir1.name, dir2.name)<EOL><DEDENT>directories.sort(sorter)<EOL>directories.reverse()<EOL><DEDENT>else:<EOL><INDENT>directories.sort(key=operator.attrgetter('<STR_LIT:name>'), reverse=True)<EOL><DEDENT>for tarinfo in directories:<EOL><INDENT>dirpath = os.path.join(path, tarinfo.name)<EOL>try:<EOL><INDENT>self.chown(tarinfo, dirpath)<EOL>self.utime(tarinfo, dirpath)<EOL>self.chmod(tarinfo, dirpath)<EOL><DEDENT>except ExtractError:<EOL><INDENT>e = sys.exc_info()[<NUM_LIT:1>]<EOL>if self.errorlevel > <NUM_LIT:1>:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>self._dbg(<NUM_LIT:1>, "<STR_LIT>" % e)<EOL><DEDENT><DEDENT><DEDENT>

Extract all members from the archive to the current working
       directory and set owner, modification time and permissions on
       directories afterwards. `path' specifies a different directory
       to extract to. `members' is optional and must be a subset of the
       list returned by getmembers().


<EOL>return self.name == oth<EOL>

Use by all __contains__ call when we do some 'in' test


def unify(self, oth_type_def: TypeExprComponent, blhs, brhs) -> TypeExprComponent:

print("<STR_LIT>" % (self, oth_type_def))<EOL>if self.type_def is not None:<EOL><INDENT>if type(oth_type_def) is UnknownName:<EOL><INDENT>if oth_type_def is None:<EOL><INDENT>oth_type_def.type_def = self.type_def<EOL>return self.type_def<EOL><DEDENT><DEDENT>return self.type_def.unify(oth_type_def, blhs, brhs)<EOL><DEDENT>if self.type_def is None:<EOL><INDENT>self.type_def = Overload()<EOL><DEDENT>if oth_type_def not in self.type_def:<EOL><INDENT>self.type_def.append(oth_type_def)<EOL><DEDENT>return oth_type_def<EOL>

When we unify an Unknown Name vs another type def we always match


print("<STR_LIT>" % (self, oth_type_def))<EOL>if type(oth_type_def) is not Fun:<EOL><INDENT>if type(oth_type_def) is T:<EOL><INDENT>return self[<NUM_LIT:0>].unify(oth_type_def, blhs, brhs)<EOL><DEDENT>raise "<STR_LIT>"<EOL><DEDENT>diff_len = len(self) - len(oth_type_def)<EOL>if diff_len < <NUM_LIT:0>: <EOL><INDENT>return None<EOL><DEDENT>for a, b in zip(reversed(self), reversed(oth_type_def)):<EOL><INDENT>if not a.unify(b, blhs, brhs):<EOL><INDENT>return None<EOL><DEDENT><DEDENT>return Fun(*self[:diff_len])<EOL>

When we unify a function vs another type def we match each term and we return the rest.

t1 -> t2 -> t3 ?? t1 -> t2 match and we return t3
t1 -> t2 ?? t1 -> t2 -> t3 didn't match

Note: the first element is the return type


@meta.rule(parsing.Parser, "<STR_LIT>")<EOL>def ignore_cxx(self) -> bool:

self._stream.save_context()<EOL>while not self.read_eof():<EOL><INDENT>idxref = self._stream.index<EOL>if self._stream.peek_char in "<STR_LIT>":<EOL><INDENT>while (not self.read_eof()<EOL>and self._stream.peek_char in "<STR_LIT>"):<EOL><INDENT>self._stream.incpos()<EOL><DEDENT><DEDENT>if self.peek_text("<STR_LIT>"):<EOL><INDENT>while not self.read_eof() and not self.peek_char("<STR_LIT:\n>"):<EOL><INDENT>self._stream.incpos()<EOL><DEDENT>if not self.read_char("<STR_LIT:\n>") and self.read_eof():<EOL><INDENT>return self._stream.validate_context()<EOL><DEDENT><DEDENT>if self.peek_text("<STR_LIT>"):<EOL><INDENT>while not self.read_eof() and not self.peek_text("<STR_LIT>"):<EOL><INDENT>self._stream.incpos()<EOL><DEDENT>if not self.read_text("<STR_LIT>") and self.read_eof():<EOL><INDENT>return self._stream.restore_context()<EOL><DEDENT><DEDENT>if idxref == self._stream.index:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return self._stream.validate_context()<EOL>

Consume comments and whitespace characters.


def catend(dst: str, src: str, indent) -> str:

res = dst<EOL>txtsrc = src<EOL>if not isinstance(src, str):<EOL><INDENT>txtsrc = str(src)<EOL><DEDENT>for c in list(txtsrc):<EOL><INDENT>if len(res) > <NUM_LIT:0> and res[-<NUM_LIT:1>] == '<STR_LIT:\n>':<EOL><INDENT>res += (indentable.char_indent * indentable.num_indent) *(indent - <NUM_LIT:1>) + c<EOL><DEDENT>else:<EOL><INDENT>res += c<EOL><DEDENT><DEDENT>return res<EOL>

cat two strings but handle \n for tabulation


def list_set_indent(lst: list, indent: int=<NUM_LIT:1>):

for i in lst:<EOL><INDENT>if isinstance(i, indentable):<EOL><INDENT>i.set_indent(indent)<EOL><DEDENT>if isinstance(i, list):<EOL><INDENT>list_set_indent(i, indent)<EOL><DEDENT><DEDENT>


def populate_from_sequence(seq: list, r: ref(Edge), sr: state.StateRegister):

base_state = r<EOL>idxlast = len(seq) - <NUM_LIT:1><EOL>idx = <NUM_LIT:0><EOL>for m in seq:<EOL><INDENT>if isinstance(m, list):<EOL><INDENT>for item in m:<EOL><INDENT>populate_from_sequence(item, r, sr)<EOL><DEDENT><DEDENT>elif isinstance(m, MatchExpr):<EOL><INDENT>eX = r().get_next_edge(m)<EOL>if eX is None:<EOL><INDENT>sX = None<EOL>if idx != idxlast:<EOL><INDENT>sX = state.State(sr)<EOL>sX.matchDefault(base_state().s)<EOL><DEDENT>else:<EOL><INDENT>sX = base_state().s<EOL><DEDENT>eX = Edge(sX)<EOL>r().next_edge[id(sX)] = eX<EOL>m.attach(r().s, sX, sr)<EOL><DEDENT>r = ref(eX)<EOL><DEDENT>idx += <NUM_LIT:1><EOL><DEDENT>

function that connect each other one sequence of MatchExpr.


def populate_state_register(all_seq: [list], sr: state.StateRegister) -> Edge:

<EOL>s0 = state.State(sr)<EOL>s0.matchDefault(s0)<EOL>sr.set_default_state(s0)<EOL>e0 = Edge(s0)<EOL>for seq in all_seq:<EOL><INDENT>r = ref(e0)<EOL>populate_from_sequence(seq, r, sr)<EOL><DEDENT>return e0<EOL>

function that create a state for all instance
        of MatchExpr in the given list and connect each others.


cmd = pipes.Template()<EOL>cmd.append('<STR_LIT>' % fname, '<STR_LIT>')<EOL>with cmd.open('<STR_LIT>', '<STR_LIT:w>') as f:<EOL><INDENT>f.write(self.to_dot())<EOL><DEDENT>


<EOL>must_delete = []<EOL>l = len(self.ls)<EOL>for idx, ls in zip(range(l), self.ls):<EOL><INDENT>ids = id(ls[<NUM_LIT:1>].thestate())<EOL>if ids == id(ls[<NUM_LIT:0>]) and (ls[<NUM_LIT:1>].have_finish or not ls[<NUM_LIT:1>].alive):<EOL><INDENT>must_delete.append(idx)<EOL><DEDENT>elif ls[<NUM_LIT:1>].alive:<EOL><INDENT>ls[<NUM_LIT:1>].alive = False<EOL><DEDENT><DEDENT>for delete in reversed(must_delete):<EOL><INDENT>self.ls.pop(delete)<EOL><DEDENT>self.init_all()<EOL>

Only one Living State on the S0 of each StateRegister


@meta.hook(EBNF, "<STR_LIT>")<EOL>def add_optional(self, repeat):

repeat.functor = parsing.RepOptional<EOL>return True<EOL>


res = None<EOL>try:<EOL><INDENT>res = self.eval_rule('<STR_LIT>')<EOL>if not res:<EOL><INDENT>self.diagnostic.notify(<EOL>error.Severity.ERROR,<EOL>"<STR_LIT>" % self._lastRule,<EOL>error.LocationInfo.from_maxstream(self._stream)<EOL>)<EOL>raise self.diagnostic<EOL><DEDENT><DEDENT>except error.Diagnostic as d:<EOL><INDENT>d.notify(<EOL>error.Severity.ERROR,<EOL>"<STR_LIT>" % self._lastRule<EOL>)<EOL>raise d<EOL><DEDENT>return res<EOL>

Parse the DSL and provide a dictionnaries of all resulting rules.
Call by the MetaGrammar class.

TODO: could be done in the rules property of parsing.BasicParser???


@meta.hook(EBNF, "<STR_LIT>")<EOL>def param_num(self, param, n):

param.pair = (int(self.value(n)), int)<EOL>return True<EOL>


@meta.hook(EBNF, "<STR_LIT>")<EOL>def add_rpt(self, sequence, mod, pt):

modstr = self.value(mod)<EOL>if modstr == '<STR_LIT>':<EOL><INDENT>self._stream.restore_context()<EOL>self.diagnostic.notify(<EOL>error.Severity.ERROR,<EOL>"<STR_LIT>",<EOL>error.LocationInfo.from_stream(self._stream, is_error=True)<EOL>)<EOL>raise self.diagnostic<EOL><DEDENT>if modstr == '<STR_LIT:!>':<EOL><INDENT>self._stream.restore_context()<EOL>self.diagnostic.notify(<EOL>error.Severity.ERROR,<EOL>"<STR_LIT>",<EOL>error.LocationInfo.from_stream(self._stream, is_error=True)<EOL>)<EOL>raise self.diagnostic<EOL><DEDENT>oldnode = sequence<EOL>sequence.parser_tree = pt.functor(oldnode.parser_tree)<EOL>return True<EOL>


@meta.hook(EBNF, "<STR_LIT>")<EOL>def param_str(self, param, s):

param.pair = (self.value(s).strip('<STR_LIT:">'), str)<EOL>return True<EOL>


@meta.hook(EBNF, "<STR_LIT>")<EOL>def add_mod(self, seq, mod):

modstr = self.value(mod)<EOL>if modstr == '<STR_LIT>':<EOL><INDENT>seq.parser_tree = parsing.Complement(seq.parser_tree)<EOL><DEDENT>elif modstr == '<STR_LIT>':<EOL><INDENT>seq.parser_tree = parsing.LookAhead(seq.parser_tree)<EOL><DEDENT>elif modstr == '<STR_LIT:!>':<EOL><INDENT>seq.parser_tree = parsing.Neg(seq.parser_tree)<EOL><DEDENT>elif modstr == '<STR_LIT>':<EOL><INDENT>seq.parser_tree = parsing.Until(seq.parser_tree)<EOL><DEDENT>return True<EOL>

Create a tree.{Complement, LookAhead, Neg, Until}


def set_one(chainmap, thing_name, callobject):

namespaces = reversed(thing_name.split("<STR_LIT:.>"))<EOL>lstname = []<EOL>for name in namespaces:<EOL><INDENT>lstname.insert(<NUM_LIT:0>, name)<EOL>strname = '<STR_LIT:.>'.join(lstname)<EOL>chainmap[strname] = callobject<EOL><DEDENT>

Add a mapping with key thing_name for callobject in chainmap with
        namespace handling.


global _directives<EOL>class_dir_list = _directives<EOL>def wrapper(f):<EOL><INDENT>nonlocal directname<EOL>if directname is None:<EOL><INDENT>directname = f.__name__<EOL><DEDENT>f.ns_name = directname<EOL>set_one(class_dir_list, directname, f)<EOL>return f<EOL><DEDENT>return wrapper<EOL>

Attach a class to a parsing class and register it as a parser directive.

        The class is registered with its name unless directname is provided.


@meta.rule(Parser, "<STR_LIT>")<EOL>def read_integer(self) -> bool:

if self.read_eof():<EOL><INDENT>return False<EOL><DEDENT>self._stream.save_context()<EOL>c = self._stream.peek_char<EOL>if c.isdigit():<EOL><INDENT>self._stream.incpos()<EOL>while not self.read_eof():<EOL><INDENT>c = self._stream.peek_char<EOL>if not c.isdigit():<EOL><INDENT>break<EOL><DEDENT>self._stream.incpos()<EOL><DEDENT>return self._stream.validate_context()<EOL><DEDENT>return self._stream.restore_context()<EOL>

read a number
Read following BNF rule else return False::

    readInteger = [
        ['0'..'9']+
    ]


return self.tag_cache[name]<EOL>


self._ignores.pop()<EOL>return True<EOL>


s = self._streams.pop()<EOL>self.clean_tmp(s)<EOL>

Pop the last Stream pushed on to the parser stack.


<EOL>self.tag_cache[name] = Tag(self._stream, self._stream.index)<EOL>return True<EOL>

Save the current index under the given name.


@meta.rule(Parser, "<STR_LIT>")<EOL>def read_hex_integer(self) -> bool:

if self.read_eof():<EOL><INDENT>return False<EOL><DEDENT>self._stream.save_context()<EOL>c = self._stream.peek_char<EOL>if c.isdigit() or ('<STR_LIT:a>' <= c.lower() and c.lower() <= '<STR_LIT:f>'):<EOL><INDENT>self._stream.incpos()<EOL>while not self.read_eof():<EOL><INDENT>c = self._stream.peek_char<EOL>if not (c.isdigit() or ('<STR_LIT:a>' <= c.lower() and c.lower() <= '<STR_LIT:f>')):<EOL><INDENT>break<EOL><DEDENT>self._stream.incpos()<EOL><DEDENT>return self._stream.validate_context()<EOL><DEDENT>return self._stream.restore_context()<EOL>

read a hexadecimal number
Read the following BNF rule else return False::

    readHexInteger = [
        [ '0'..'9' | 'a'..'f' | 'A'..'F' ]+
    ]


if self.read_eof():<EOL><INDENT>return False<EOL><DEDENT>self._stream.save_context()<EOL>if c == self._stream.peek_char:<EOL><INDENT>self._stream.incpos()<EOL>return self._stream.validate_context()<EOL><DEDENT>return self._stream.restore_context()<EOL>

Consume the c head byte, increment current index and return True
else return False. It use peekchar and it's the same as '' in BNF.


@classmethod<EOL><INDENT>def set_directives(cls, directives: dict) -> bool:<DEDENT>

meta._directives = meta._directives.new_child()<EOL>for dir_name, dir_pt in directives.items():<EOL><INDENT>meta.set_one(meta._directives, dir_name, dir_pt)<EOL>dir_pt.ns_name = dir_name<EOL><DEDENT>return True<EOL>

Merge internal directives set with the given directives.
For working directives, attach it only in the dsl.Parser class


@classmethod<EOL><INDENT>def set_rules(cls, rules: dict) -> bool:<DEDENT>

cls._rules = cls._rules.new_child()<EOL>for rule_name, rule_pt in rules.items():<EOL><INDENT>if '<STR_LIT:.>' not in rule_name:<EOL><INDENT>rule_name = cls.__module__+ '<STR_LIT:.>' + cls.__name__+ '<STR_LIT:.>' + rule_name<EOL><DEDENT>meta.set_one(cls._rules, rule_name, rule_pt)<EOL><DEDENT>return True<EOL>

Merge internal rules set with the given rules


@property<EOL><INDENT>def nstream(self) -> int:<DEDENT>

return len(self._streams)<EOL>


self._stream.save_context()<EOL>if not self.read_eof() and self._stream.peek_char in "<STR_LIT>":<EOL><INDENT>while (not self.read_eof()<EOL>and self._stream.peek_char in "<STR_LIT>"):<EOL><INDENT>self._stream.incpos()<EOL><DEDENT>return self._stream.validate_context()<EOL><DEDENT>return self._stream.validate_context()<EOL>


def eval_hook(self, name: str, ctx: list) -> Node:

if name not in self.__class__._hooks:<EOL><INDENT>self.diagnostic.notify(<EOL>error.Severity.ERROR,<EOL>"<STR_LIT>" % name,<EOL>error.LocationInfo.from_stream(self._stream, is_error=True)<EOL>)<EOL>raise self.diagnostic<EOL><DEDENT>self._lastRule = '<STR_LIT:#>' + name<EOL>res = self.__class__._hooks[name](self, *ctx)<EOL>if type(res) is not bool:<EOL><INDENT>raise TypeError("<STR_LIT>" % name)<EOL><DEDENT>return res<EOL>


if self.read_eof():<EOL><INDENT>return True<EOL><DEDENT>self._stream.save_context()<EOL>while not self.read_eof():<EOL><INDENT>self._stream.incpos()<EOL><DEDENT>return self._stream.validate_context()<EOL>

Consume all the stream. Same as EOF in BNF.


@meta.rule(BasicParser, "<STR_LIT>")<EOL>def read_eof(self) -> bool:

<EOL>return self._stream.index == self._stream.eos_index<EOL>

Returns true if reached end of the stream.


start = self._stream.index<EOL>stop = start + len(text)<EOL>if stop > self._stream.eos_index:<EOL><INDENT>return False<EOL><DEDENT>return self._stream[self._stream.index:stop] == text<EOL>

Same as readText but doesn't consume the stream.


@property<EOL><INDENT>def max_readed_position(self) -> Position:<DEDENT>

return Position(self._maxindex, self._maxline, self._maxcol)<EOL>

The index of the deepest character readed.


@property<EOL><INDENT>def eos_index(self) -> int:<DEDENT>

return self._len<EOL>


del self._contexts[-<NUM_LIT:1>]<EOL>return True<EOL>


self._cursor.position = self._contexts.pop()<EOL>return False<EOL>


@property<EOL><INDENT>def last_readed_line(self) -> str:<DEDENT>

mpos = self._cursor.max_readed_position<EOL>mindex = mpos.index<EOL>prevline = mindex - <NUM_LIT:1> if mindex == self.eos_index else mindex<EOL>while prevline >= <NUM_LIT:0> and self._content[prevline] != '<STR_LIT:\n>':<EOL><INDENT>prevline -= <NUM_LIT:1><EOL><DEDENT>nextline = mindex<EOL>while nextline < self.eos_index and self._content[nextline] != '<STR_LIT:\n>':<EOL><INDENT>nextline += <NUM_LIT:1><EOL><DEDENT>last_line = self._content[prevline + <NUM_LIT:1>:nextline]<EOL>return last_line<EOL>

Usefull string to compute error message.


<EOL>if len(self._eol) > <NUM_LIT:0>:<EOL><INDENT>self.position = self._eol.pop()<EOL><DEDENT>


@property<EOL><INDENT>def position(self) -> Position:<DEDENT>

return Position(self._index, self._lineno, self._col_offset)<EOL>


def check(self, ndict: dict, info="<STR_LIT>") -> bool:

def iscycle(thing, ndict: dict, info: str) -> bool:<EOL><INDENT>idthing = id(thing)<EOL>ndict[info] = idthing<EOL>if idthing not in ndict:<EOL><INDENT>ndict[idthing] = "<STR_LIT>" % (type(thing), info)<EOL>return False<EOL><DEDENT>else:<EOL><INDENT>ndict[idthing] += "<STR_LIT>" % (type(thing), info)<EOL>return True<EOL><DEDENT><DEDENT>def recurs(thing, ndict: dict, info: str) -> bool:<EOL><INDENT>if not iscycle(thing, ndict, info):<EOL><INDENT>res = False<EOL>if isinstance(thing, list):<EOL><INDENT>idx = <NUM_LIT:0><EOL>for i in thing:<EOL><INDENT>res |= recurs(i, ndict, "<STR_LIT>" % (info, idx))<EOL>idx += <NUM_LIT:1><EOL><DEDENT><DEDENT>elif isinstance(thing, Node):<EOL><INDENT>res |= thing.check(ndict, info)<EOL><DEDENT>elif isinstance(thing, dict):<EOL><INDENT>for k, v in thing.items():<EOL><INDENT>res |= recurs(v, ndict, "<STR_LIT>" % (info, k))<EOL><DEDENT><DEDENT>return res<EOL><DEDENT>return True<EOL><DEDENT>if len(ndict) == <NUM_LIT:0>:<EOL><INDENT>ndict['<STR_LIT>'] = id(self)<EOL>info = '<STR_LIT>'<EOL><DEDENT>if not iscycle(self, ndict, info):<EOL><INDENT>res = False<EOL>if len(self) > <NUM_LIT:0>:<EOL><INDENT>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>keys = list(self.keys())<EOL>for k in keys:<EOL><INDENT>ndict["<STR_LIT:[>" + repr(k) + "<STR_LIT:]>"] = id(self[k])<EOL>res |= recurs(self[k], ndict, "<STR_LIT>" % (info, k))<EOL><DEDENT><DEDENT><DEDENT>keys = list(vars(self).keys())<EOL>for k in keys:<EOL><INDENT>ndict["<STR_LIT:.>" + k] = id(getattr(self, k))<EOL>res |= recurs(getattr(self, k), ndict, "<STR_LIT>" % (info, k))<EOL><DEDENT>return res<EOL><DEDENT>return True<EOL>

Debug method, help detect cycle and/or
other incoherence in a tree of Node


tmp = self<EOL>while tmp is not None:<EOL><INDENT>yield tmp.data<EOL>tmp = tmp.next<EOL><DEDENT>


def after_parse(self, node: parsing.Node) -> parsing.Node:

return node<EOL>

If you want to do some stuff after parsing, overload this...


@meta.hook(BasicParser, "<STR_LIT>")<EOL>def pred_neq(self, n, val):

v1 = n.value<EOL>v2 = val<EOL>if hasattr(val, '<STR_LIT:value>'):<EOL><INDENT>v2 = val.value<EOL><DEDENT>if isinstance(v1, int) and not isinstance(v2, int):<EOL><INDENT>return v1 != int(v2)<EOL><DEDENT>return v1 != v2<EOL>

Test if a node set with setint or setstr not equal a certain value

example::

    R = [
        __scope__:n
        ['a' #setint(n, 12) | 'b' #setint(n, 14)]
        C
        [#neq(n, 12) D]
    ]


print("<STR_LIT>")<EOL>try:<EOL><INDENT>print("<STR_LIT>")<EOL>for k, v in self.id_cache.items():<EOL><INDENT>print("<STR_LIT>" % (k, v))<EOL><DEDENT>print("<STR_LIT>")<EOL>for k, v in self.tag_cache.items():<EOL><INDENT>print("<STR_LIT>" % (k, v))<EOL><DEDENT>print("<STR_LIT>")<EOL>for k, v in self.rule_nodes.items():<EOL><INDENT>txt = "<STR_LIT>" % (k, id(v))<EOL>if k in self.tag_cache:<EOL><INDENT>tag = self.tag_cache[k]<EOL>txt += "<STR_LIT>" % tag<EOL>k = "<STR_LIT>" % (tag._begin, tag._end)<EOL>if k in self._stream.value_cache:<EOL><INDENT>txt += "<STR_LIT>" % self._stream.value_cache[k]<EOL><DEDENT><DEDENT>print(txt)<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>print("<STR_LIT>" % err)<EOL><DEDENT>import sys<EOL>sys.stdout.flush()<EOL>return True<EOL>

Dump tag,rule,id and value cache. For debug.

example::

    R = [
        #dump_nodes
    ]


@meta.hook(BasicParser, "<STR_LIT>")<EOL>def set_node_as_int(self, dst, src):

dst.value = self.value(src)<EOL>return True<EOL>

Set a node to a value captured from another node

example::

    R = [
        In : node #setcapture(_, node)
    ]


@meta.hook(BasicParser, "<STR_LIT>")<EOL>def get_subnode(self, dst, ast, expr):

dst.value = eval('<STR_LIT>' + expr)<EOL>return True<EOL>

get the value of subnode

example::

    R = [
        __scope__:big  getsomethingbig:>big
        #get(_, big, '.val') // copy big.val into _
    ]


@meta.add_method(functors.SkipIgnore)<EOL>def to_cython(self, genstate) -> fmt.indentable:

return '<STR_LIT:\n>'<EOL>

TODO: A quite important part of C transform... rethink directive/decoration/etc...


@meta.add_method(Translator)<EOL>def to_fmt(self, with_from=False) -> fmt.indentable:

Return a Fmt representation of Translator for pretty-printing


def get_resolved_names(self, type_name: TypeName) -> list:

if not isinstance(type_name, TypeName):<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% type(type_name))<EOL><DEDENT>rnames = []<EOL>for name in type_name.components:<EOL><INDENT>if name not in self.resolution:<EOL><INDENT>raise Exception("<STR_LIT>" % name)<EOL><DEDENT>rname = self.resolution[name]<EOL>if rname is not None:<EOL><INDENT>rname = rname().show_name()<EOL><DEDENT>else:<EOL><INDENT>rname = name<EOL><DEDENT>rnames.append(rname)<EOL><DEDENT>return rnames<EOL>

Use self.resolution to subsitute type_name.
Allow to instanciate polymorphic type ?1, ?toto


ret = self<EOL>if parent is not None:<EOL><INDENT>ret = self._sig.set_parent(parent)<EOL>self.resolve()<EOL><DEDENT>elif not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.parent = None<EOL><DEDENT>return ret<EOL>

When we add a parent (from Symbol), don't forget to resolve.


def use_variadic_types(self, list_type: [TypeName]):

self._variadic_types = list_type<EOL>self.resolve()<EOL>

Attach a list of types for extra variadic argument of a call


return self._sig.set_parent(parent)<EOL>


unq = super().internal_name()<EOL>if self.tret is not None:<EOL><INDENT>unq += "<STR_LIT:_>" + self.tret<EOL><DEDENT>return unq<EOL>


state = self.__dict__.copy()<EOL>del state['<STR_LIT>']<EOL>return state<EOL>


<EOL>lstparent = [self]<EOL>p = self.get_parent()<EOL>while p is not None:<EOL><INDENT>lstparent.append(p)<EOL>p = p.get_parent()<EOL><DEDENT>return lstparent<EOL>

Return the list of all contained scope from global to local


unq = "<STR_LIT:_>".join(self.get_scope_names())<EOL>return unq<EOL>


def infer_block(self, body, diagnostic=None):

<EOL>for e in body:<EOL><INDENT>e.infer_node = InferNode(parent=self.infer_node)<EOL>e.infer_type(diagnostic=diagnostic)<EOL><DEDENT>

Infer type on block is to type each of is sub-element


def infer_subexpr(self, expr, diagnostic=None):

expr.infer_node = InferNode(parent=self.infer_node)<EOL>expr.infer_type(diagnostic=diagnostic)<EOL>


def type_algos(self) -> ('<STR_LIT>',<EOL>'<STR_LIT>', ['<STR_LIT:args>']):

raise Exception(("<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>") % type(self).__name__)<EOL>

Sub class must return a Tuple of 3 elements:
    - the method to use to infer type.
    - the list of params to used when infer a type.
    - the method to use when feedback a type.

This is useful to connect AST members to generic algo or
to overload some specific semantic for your language.


def __isub__(self, sig: Scope) -> Scope:

return self.difference_update(sig)<EOL>


def symmetric_difference_update(self, oset: Scope) -> Scope:

skey = set()<EOL>keys = list(self._hsig.keys())<EOL>for k in keys:<EOL><INDENT>if k in oset:<EOL><INDENT>skey.add(k)<EOL><DEDENT><DEDENT>for k in oset._hsig.keys():<EOL><INDENT>if k not in skey:<EOL><INDENT>self._hsig[k] = oset.get(k)<EOL><DEDENT><DEDENT>for k in skey:<EOL><INDENT>del self._hsig[k]<EOL><DEDENT>return self<EOL>

Remove common values
            and Update specific values from another Set


new = Scope(sig=self._hsig.values(), state=self.state)<EOL>new |= sig<EOL>return new<EOL>

Create a new Set produce by the union of 2 Set


def __init__(self, name: str=None, sig: [Signature]=None,<EOL>state=StateScope.FREE, is_namespace=True):

if name is not None and not isinstance(name, str):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>super().__init__(name)<EOL>self.need_feedback = False<EOL>self.state = state<EOL>self.is_namespace = is_namespace<EOL>self.mapTypeTranslate = MapSourceTranslate()<EOL>self.astTranslatorInjector = None<EOL>self._ntypes = <NUM_LIT:0><EOL>self._nvars = <NUM_LIT:0><EOL>self._nfuns = <NUM_LIT:0><EOL>self._hsig = {}<EOL>if sig is not None:<EOL><INDENT>if isinstance(sig, Signature) or isinstance(sig, Scope):<EOL><INDENT>self.add(sig)<EOL><DEDENT>elif len(sig) > <NUM_LIT:0>:<EOL><INDENT>self.update(sig)<EOL><DEDENT><DEDENT>

Unnamed scope for global scope

        A Scope have basically 3 possibles states:

            FREE: it's a standalone Scope, generally the global Scope
            LINKED: the Scope is related to another Scope for type resolution,
                like compound statement
            EMBEDDED: the Scope was added into another Scope,
                it forwards all 'in' calls...like namespacing

        A Scope could or couldn't act like a namespace.
        All Signature added into a 'namespaced' Scope was prefixed
        by the Scope name.


return self.symmetric_difference(sig)<EOL>


return repr(self._hsig)<EOL>


if self.state == StateScope.EMBEDDED and self.parent is not None:<EOL><INDENT>return list(self._hsig.values()) + list(self.parent().values())<EOL><DEDENT>else:<EOL><INDENT>return self._hsig.values()<EOL><DEDENT>


def get_all_polymorphic_return(self) -> bool:

lst = []<EOL>for s in self.values():<EOL><INDENT>if hasattr(s, '<STR_LIT>') and s.tret.is_polymorphic:<EOL><INDENT>lst.append(EvalCtx.from_sig(s))<EOL><DEDENT><DEDENT>rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)<EOL>rscope.set_parent(self)<EOL>return rscope<EOL>

For now, polymorphic return type are handle by symbol artefact.

        --> possible multi-polymorphic but with different constraint attached!


return self.intersection(sig)<EOL>


def remove(self, it: Signature) -> bool:

txt = it.internal_name()<EOL>if txt not in self._hsig:<EOL><INDENT>raise KeyError(it.show_name() + '<STR_LIT>')<EOL><DEDENT>sig = self._hsig[txt]<EOL>if isinstance(sig, Scope):<EOL><INDENT>sig.state = StateScope.LINKED<EOL><DEDENT>del self._hsig[txt]<EOL>return True<EOL>

Remove it but raise KeyError if not found


def __contains__(self, s: Signature) -> bool:

<EOL>from pyrser.type_system.type import Type<EOL>found = False<EOL>txt = "<STR_LIT>"<EOL>if isinstance(s, Signature):<EOL><INDENT>txt = s.internal_name()<EOL><DEDENT>elif isinstance(s, Type):<EOL><INDENT>txt = s.type_name<EOL><DEDENT>elif isinstance(s, str):<EOL><INDENT>txt = s<EOL><DEDENT>found = (txt in self._hsig)<EOL>if not found and self.parent is not None:<EOL><INDENT>if self.state != StateScope.FREE and isinstance(s, Type):<EOL><INDENT>found = (s.type_name in self.parent())<EOL><DEDENT>if self.state == StateScope.EMBEDDED:<EOL><INDENT>found = (txt in self.parent())<EOL><DEDENT><DEDENT>return found<EOL>

check if a Signature or a Type is declared in a Scope


if isinstance(it, Scope):<EOL><INDENT>it.state = StateScope.EMBEDDED<EOL><DEDENT>txt = it.internal_name()<EOL>it.set_parent(self)<EOL>if self.is_namespace:<EOL><INDENT>txt = it.internal_name()<EOL><DEDENT>if txt == "<STR_LIT>":<EOL><INDENT>txt = '<STR_LIT:_>' + str(len(self._hsig))<EOL><DEDENT>if txt in self._hsig:<EOL><INDENT>raise KeyError("<STR_LIT>" % txt)<EOL><DEDENT>self._hsig[txt] = it<EOL>self.__update_count()<EOL>return True<EOL>


self._ntypes = self.count_types()<EOL>self._nvars = self.count_vars()<EOL>self._nfuns = self.count_funs()<EOL>


pp = fmt.tab([])<EOL>to_yml_item(self, pp.lsdata, "<STR_LIT>")<EOL>return str(pp)<EOL>

Allow to get the YML string representation of a Node.::

    from pyrser.passes import to_yml

    t = Node()
    ...
    print(str(t.to_yml()))


def visit_Seq(self, node: parsing.Seq) -> [ast.stmt] or ast.expr:

exprs, stmts = [], []<EOL>for clause in node.ptlist:<EOL><INDENT>clause_ast = self.visit(clause)<EOL>if isinstance(clause_ast, ast.expr):<EOL><INDENT>exprs.append(clause_ast)<EOL><DEDENT>else:<EOL><INDENT>if exprs:<EOL><INDENT>stmts.extend(self.combine_exprs_for_clauses(exprs))<EOL>exprs = []<EOL><DEDENT>stmts.extend(self._clause(clause_ast))<EOL><DEDENT><DEDENT>if not stmts:<EOL><INDENT>return ast.BoolOp(ast.And(), exprs)<EOL><DEDENT>if exprs:<EOL><INDENT>stmts.extend(self.combine_exprs_for_clauses(exprs))<EOL><DEDENT>return stmts<EOL>

Generates python code for clauses.

        #Continuous clauses which can can be inlined are combined with and
        clause and clause

        if not clause:
            return False
        if not clause:
            return False


if self.in_optional:<EOL><INDENT>return ast.Pass()<EOL><DEDENT>if self.in_try:<EOL><INDENT>return ast.Raise(<EOL>ast.Call(ast.Name('<STR_LIT>', ast.Load()), [], [], None, None),<EOL>None)<EOL><DEDENT>if self.in_loop:<EOL><INDENT>return ast.Break()<EOL><DEDENT>return ast.Return(ast.Name('<STR_LIT:False>', ast.Load()))<EOL>

Create the appropriate scope exiting statement.

        The documentation only shows one level and always uses
        'return False' in examples.

        'raise AltFalse()' within a try.
        'break' within a loop.
        'return False' otherwise.


def visit_CallTrue(self, node: parsing.CallTrue) -> ast.expr:

return ast.Lambda(<EOL>ast.arguments([], None, None, [], None, None, [], []),<EOL>ast.BoolOp(<EOL>ast.Or(),<EOL>[<EOL>self.visit_Call(node),<EOL>ast.Name('<STR_LIT:True>', ast.Load())]))<EOL>

Generates python code calling the function and returning True.

        lambda: fn(*args) or True


def visit_Call(self, node: parsing.Call) -> ast.expr:

return ast.Call(<EOL>ast.Attribute(<EOL>ast.Name('<STR_LIT>', ast.Load),<EOL>node.callObject.__name__,<EOL>ast.Load()),<EOL>[ast.Str(param) for param in node.params],<EOL>[],<EOL>None,<EOL>None)<EOL>

Generates python code calling the function.

        fn(*args)


def visit_Rule(self, node: parsing.Rule) -> ast.expr:

return ast.Call(<EOL>ast.Attribute(ast.Name('<STR_LIT>', ast.Load()),<EOL>'<STR_LIT>', ast.Load()),<EOL>[ast.Str(node.name)], [], None, None)<EOL>

Generates python code calling a rule.

        self.evalRule('rulename')


def visit_Rep1N(self, node: parsing.Rep0N) -> [ast.stmt]:

clause = self.visit(node.pt)<EOL>if isinstance(clause, ast.expr):<EOL><INDENT>return (self._clause(clause) + self.visit_Rep0N(node))<EOL><DEDENT>self.in_loop += <NUM_LIT:1><EOL>clause = self._clause(self.visit(node.pt))<EOL>self.in_loop -= <NUM_LIT:1><EOL>return self._clause(self.visit(node.pt)) + [<EOL>ast.While(ast.Name('<STR_LIT:True>', ast.Load()), clause, [])]<EOL>

Generates python code for a clause repeated 1 or more times.

        <code for the clause>
        while True:
            <code for the clause>


if tag.name in ATTR_WHITELIST.keys():<EOL><INDENT>attrs = copy(tag.attrs)<EOL>for attr, value in attrs.items():<EOL><INDENT>if attr in ATTR_WHITELIST[tag.name]:<EOL><INDENT>tag.attrs[attr] = self._parse_attr(tag.name, attr, value)<EOL><DEDENT>else:<EOL><INDENT>del tag.attrs[attr]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>tag.attrs = {}<EOL><DEDENT>

Reject attributes not defined in ATTR_WHITELIST.


if tag.get('<STR_LIT:id>', '<STR_LIT>').startswith('<STR_LIT>'):<EOL><INDENT>tag.parent.extract()<EOL><DEDENT>


tag.unwrap()<EOL>

Remove span tags while preserving contents.


style = tag.get('<STR_LIT>')<EOL>if style and '<STR_LIT>' in style:<EOL><INDENT>tag.wrap(self.soup.new_tag('<STR_LIT>'))<EOL><DEDENT>

See if span tag has italic style and wrap with em tag.


def _parse_attr(self, tagname, attr, value):

if tagname == '<STR_LIT:a>' and attr == '<STR_LIT>':<EOL><INDENT>return self._parse_href(value)<EOL><DEDENT>else:<EOL><INDENT>return value<EOL><DEDENT>

Parse attribute. Delegate to href parser for hrefs, otherwise return
value.


pkg = Package("<STR_LIT>")<EOL>class Container(Injector):<EOL><INDENT>foo = pkg.injected.Container.foo<EOL>bar = pkg.injected.Container.bar<EOL>baz = <NUM_LIT:2><EOL><DEDENT>return Container<EOL>


pkg = Package("<STR_LIT>")<EOL>sub = Package("<STR_LIT>")<EOL>class Container(Injector):<EOL><INDENT>itself = pkg<EOL>submodule = sub<EOL>instance = sub.Bar<EOL>instance_method = sub.Bar.do<EOL>function = sub.function<EOL>variable = sub.variable<EOL>keep_class = sub.Bar<EOL>a = -<NUM_LIT:1><EOL>b = <NUM_LIT:2><EOL><DEDENT>return Container<EOL>


injected = Package("<STR_LIT>")<EOL>class Container(Injector):<EOL><INDENT>foo = injected.Container.foo<EOL>bar = injected.Container.bar<EOL>baz = <NUM_LIT:2><EOL><DEDENT>return Container<EOL>


class Container(Injector):<EOL><INDENT>foo = {("<STR_LIT:x>", <NUM_LIT:1>): <NUM_LIT:1>}<EOL>bar = this.foo[("<STR_LIT:x>", <NUM_LIT:1>)]<EOL><DEDENT>result = Container.bar<EOL>return result<EOL>


Injector.let(SubContainer=Injector.let(foo=(this << <NUM_LIT:2>).bar)).SubContainer.foo<EOL>


Injector.let(foo=(this << <NUM_LIT:1>).bar).foo<EOL>


class Container(Injector):<EOL><INDENT>foo = this.bar()<EOL>bar = lambda: <NUM_LIT:1>  <EOL><DEDENT>return Container<EOL>


class Container(Injector):<EOL><INDENT>foo = {"<STR_LIT:bar>": {"<STR_LIT>": <NUM_LIT:1>}}<EOL>class SubContainer(Injector):<EOL><INDENT>class SubSubContainer(Injector):<EOL><INDENT>spam = (this << <NUM_LIT:2>).foo["<STR_LIT:bar>"]["<STR_LIT>"]<EOL><DEDENT><DEDENT><DEDENT>result = Container.SubContainer.SubSubContainer.spam<EOL>return result<EOL>

Get item from the outer container of any depth level.


class Container(Injector):<EOL><INDENT>foo = this.bar<EOL><DEDENT>Container.foo<EOL>


this << <NUM_LIT:0><EOL>


@contrib.shared_task<EOL>class Container(Injector):<EOL><INDENT>name = "<STR_LIT>"<EOL>run = Run<EOL><DEDENT>return Container<EOL>


class NewContainer(container):<EOL><INDENT>options = {"<STR_LIT>": <NUM_LIT:10>}<EOL><DEDENT>sign = NewContainer.si(<NUM_LIT:2>, <NUM_LIT:2>, debug=True)<EOL>return sign<EOL>


sign = container.signature((<NUM_LIT:2>, <NUM_LIT:2>), {"<STR_LIT>": True}, immutable=True, countdown=<NUM_LIT:10>)<EOL>return sign<EOL>


return Injector.let(bar=(this << <NUM_LIT:2>).foo)<EOL>


class SubContainer(Injector):<EOL><INDENT>bar = (this << <NUM_LIT:1>).foo<EOL><DEDENT>return SubContainer<EOL>


Container = Injector.let()<EOL>Container.foo = <NUM_LIT:1><EOL>


class Foo(Injector):<EOL><INDENT>pass<EOL><DEDENT>Foo.let().test<EOL>


@inheritance_order<EOL>def aa10c7747a1f(Container1, Container2, Container3):

class Foo(Container1, Container2, Container3):<EOL><INDENT>pass<EOL><DEDENT>assert Foo.x == <NUM_LIT:1><EOL>


class Summator(Injector):<EOL><INDENT>foo = Foo<EOL>kwargs = {"<STR_LIT:start>": <NUM_LIT:5>}<EOL><DEDENT>


Injector.let(foo=Foo, args=(<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>))<EOL>


return Injector.let(foo=<NUM_LIT:1>)<EOL>


Foo = Injector.let()<EOL>Foo.test<EOL>


del Injector.let<EOL>

Delete attribute from `Injector` directly.


Injector.let(foo=Foo, args=(<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>), kwargs={"<STR_LIT:start>": <NUM_LIT:5>})<EOL>


@multiple_inheritance<EOL>def efdc426cd096(Foo, FooContainer, BarContainer, BazContainer):

assert isinstance((FooContainer & BarContainer & BazContainer).baz.bar.foo, Foo)<EOL>


class Bar(object):<EOL><INDENT>def __init__(self, test, two=<NUM_LIT:2>):<EOL><INDENT>self.test = test<EOL>self.two = two<EOL><DEDENT><DEDENT>class Foo(Injector):<EOL><INDENT>bar = Bar<EOL><DEDENT>Foo.bar<EOL>


class Container(Injector):<EOL><INDENT>bar = Bar<EOL><DEDENT>


class Foo(Injector):<EOL><INDENT>pass<EOL><DEDENT>Foo()<EOL>


Injector()<EOL>


class SubSubContainer(Injector):<EOL><INDENT>bar = (this << <NUM_LIT:2>).foo<EOL><DEDENT>return SubSubContainer<EOL>


class Container(Injector):<EOL><INDENT>foo = this.SubContainer.baz<EOL>SubContainer = middle<EOL><DEDENT>Container.foo<EOL>


Injector.let(foo=this.foo).foo<EOL>


return Injector.let(bar=(this << <NUM_LIT:1>).foo)<EOL>


Injector.let(foo=this.SubContainer.SubSubContainer.bar, SubContainer=middle).foo<EOL>


SubContainer = Injector.let(foo=(this << <NUM_LIT:1>).bar["<STR_LIT>"].foo)<EOL>Injector.let(SubContainer=SubContainer, bar={"<STR_LIT>": SubContainer}).SubContainer.foo<EOL>


return Injector.let(baz=(this << <NUM_LIT:1>).SubContainer1.bar)<EOL>


pkg = Package("<STR_LIT>")<EOL>return pkg.injected.SubSubContainer<EOL>


Injector.let(foo=this.bar, bar=this.foo).foo<EOL>


class Container(Injector):<EOL><INDENT>func = arg<EOL><DEDENT>


class First(Injector):<EOL><INDENT>foo = Foo<EOL><DEDENT>class Second(First):<EOL><INDENT>bar = Bar<EOL><DEDENT>Second.foo<EOL>


@value<EOL>def Foo(foo):<EOL><INDENT>pass<EOL><DEDENT>return Foo<EOL>


Summator = Injector.let(foo=Foo, bar=Bar)<EOL>Summator.foo<EOL>


class Baz(object):<EOL><INDENT>def __init__(self, foo):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return Baz<EOL>


@operation<EOL>def Bar(baz):<EOL><INDENT>pass<EOL><DEDENT>return Bar<EOL>


@operation<EOL>def Baz(foo):<EOL><INDENT>pass<EOL><DEDENT>return Baz<EOL>


pkg = Package("<STR_LIT>")<EOL>return pkg.circles.complex_value.Foo<EOL>


pkg = Package("<STR_LIT>")<EOL>return pkg.circles.simple_class.Foo<EOL>


Summator = Injector.let(foo=Foo)<EOL>Summator.foo<EOL>


class Bar(object):<EOL><INDENT>def __init__(self, foo):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return Bar<EOL>


pkg = Package("<STR_LIT>")<EOL>return pkg.circles.complex_class.Bar<EOL>


@value<EOL>def Bar(baz):<EOL><INDENT>pass<EOL><DEDENT>return Bar<EOL>


@operation<EOL>def Foo(foo):<EOL><INDENT>pass<EOL><DEDENT>return Foo<EOL>


@long_circle_deps<EOL>def fc13db5b9fda(Foo, Bar, Baz):

class First(Injector):<EOL><INDENT>foo = Foo<EOL><DEDENT>class Second(First):<EOL><INDENT>bar = Bar<EOL>baz = Baz<EOL><DEDENT>Second.foo<EOL>


Injector.let(func=arg)<EOL>


if "<STR_LIT>" not in injector:<EOL><INDENT>injector.fixture<EOL><DEDENT>options = {"<STR_LIT:name>": injector.name}<EOL>for option in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>if option in injector:<EOL><INDENT>options[option] = getattr(injector, option)<EOL><DEDENT><DEDENT>@pytest.fixture(**options)<EOL>def __fixture(request):<EOL><INDENT>return injector.let(request=request).fixture<EOL><DEDENT>__fixture.injector = injector<EOL>return __fixture<EOL>

Register Py.test fixture performing injection in it's scope.


handler = create_handler(APIView, injector)<EOL>apply_http_methods(handler, injector)<EOL>apply_api_view_methods(handler, injector)<EOL>return injector.let(as_view=handler.as_view)<EOL>

Create DRF class-based API view from injector class.


handler = create_handler(ModelViewSet, injector)<EOL>apply_api_view_methods(handler, injector)<EOL>apply_generic_api_view_methods(handler, injector)<EOL>apply_model_view_set_methods(handler, injector)<EOL>return injector.let(as_viewset=lambda: handler)<EOL>

Create DRF model view set from injector class.


return type(cls.__name__, (cls,), kwargs)<EOL>

Produce new Injector with some dependencies overwritten.


ngram_set = set()<EOL>text_length = len(text)<EOL>max_index_ngram_start = text_length - n<EOL>for i in range(max_index_ngram_start + <NUM_LIT:1>):<EOL><INDENT>ngram_set.add(tuple(text[i:i + n]))<EOL><DEDENT>return ngram_set<EOL>

Calcualtes n-grams.

    Args:
      n: which n-grams to calculate
      text: An array of tokens

    Returns:
      A set of n-grams


if len(evaluated_sentences) <= <NUM_LIT:0> or len(reference_sentences) <= <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>m = len(set(_split_into_words(reference_sentences)))<EOL>n = len(set(_split_into_words(evaluated_sentences)))<EOL>union_lcs_sum_across_all_references = <NUM_LIT:0><EOL>union = set()<EOL>for ref_s in reference_sentences:<EOL><INDENT>lcs_count, union = _union_lcs(evaluated_sentences,<EOL>ref_s,<EOL>prev_union=union)<EOL>union_lcs_sum_across_all_references += lcs_count<EOL><DEDENT>llcs = union_lcs_sum_across_all_references<EOL>r_lcs = llcs / m<EOL>p_lcs = llcs / n<EOL>beta = p_lcs / (r_lcs + <NUM_LIT>)<EOL>num = (<NUM_LIT:1> + (beta**<NUM_LIT:2>)) * r_lcs * p_lcs<EOL>denom = r_lcs + ((beta**<NUM_LIT:2>) * p_lcs)<EOL>f_lcs = num / (denom + <NUM_LIT>)<EOL>return {"<STR_LIT:f>": f_lcs, "<STR_LIT:p>": p_lcs, "<STR_LIT:r>": r_lcs}<EOL>

Computes ROUGE-L (summary level) of two text collections of sentences.
http://research.microsoft.com/en-us/um/people/cyl/download/papers/
rouge-working-note-v1.3.1.pdf

Calculated according to:
R_lcs = SUM(1, u)[LCS<union>(r_i,C)]/m
P_lcs = SUM(1, u)[LCS<union>(r_i,C)]/n
F_lcs = ((1 + beta^2)*R_lcs*P_lcs) / (R_lcs + (beta^2) * P_lcs)

where:
SUM(i,u) = SUM from i through u
u = number of sentences in reference summary
C = Candidate summary made up of v sentences
m = number of words in reference summary
n = number of words in candidate summary

Args:
  evaluated_sentences: The sentences that have been picked by the
                       summarizer
  reference_sentence: One of the sentences in the reference summaries

Returns:
  A float: F_lcs

Raises:
  ValueError: raises exception if a param has len <= 0


def _union_lcs(evaluated_sentences, reference_sentence, prev_union=None):

if prev_union is None:<EOL><INDENT>prev_union = set()<EOL><DEDENT>if len(evaluated_sentences) <= <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>lcs_union = prev_union<EOL>prev_count = len(prev_union)<EOL>reference_words = _split_into_words([reference_sentence])<EOL>combined_lcs_length = <NUM_LIT:0><EOL>for eval_s in evaluated_sentences:<EOL><INDENT>evaluated_words = _split_into_words([eval_s])<EOL>lcs = set(_recon_lcs(reference_words, evaluated_words))<EOL>combined_lcs_length += len(lcs)<EOL>lcs_union = lcs_union.union(lcs)<EOL><DEDENT>new_lcs_count = len(lcs_union) - prev_count<EOL>return new_lcs_count, lcs_union<EOL>

Returns LCS_u(r_i, C) which is the LCS score of the union longest common
subsequence between reference sentence ri and candidate summary C.
For example:
if r_i= w1 w2 w3 w4 w5, and C contains two sentences: c1 = w1 w2 w6 w7 w8
and c2 = w1 w3 w8 w9 w5, then the longest common subsequence of r_i and c1
is "w1 w2" and the longest common subsequence of r_i and c2 is "w1 w3 w5".
The union longest common subsequence of r_i, c1, and c2 is "w1 w2 w3 w5"
and LCS_u(r_i, C) = 4/5.

Args:
  evaluated_sentences: The sentences that have been picked by the
                       summarizer
  reference_sentence: One of the sentences in the reference summaries

Returns:
  float: LCS_u(r_i, C)

ValueError:
  Raises exception if a param has len <= 0


if path not in self.paths:<EOL><INDENT>self.paths.append(path)<EOL><DEDENT>

Adds a path to search through when attempting to look up a module.

:param path: the path the add to the list of searchable paths


if module_name != self.module_name:<EOL><INDENT>raise LoaderError(<EOL>'<STR_LIT>')<EOL><DEDENT>if module_name in sys.modules:<EOL><INDENT>return sys.modules[module_name]<EOL><DEDENT>module = self.load_module_py_path(module_name, self.load_target)<EOL>if self.is_pkg:<EOL><INDENT>module.__path__ = [self.module_path]<EOL>module.__package__ = module_name<EOL><DEDENT>else:<EOL><INDENT>module.__package__ = module_name.rpartition('<STR_LIT:.>')[<NUM_LIT:0>]<EOL><DEDENT>sys.modules[module_name] = module<EOL>return module<EOL>

Loads a module's code and sets the module's expected hidden
variables. For more information on these variables and what they
are for, please see PEP302.

:param module_name: the full name of the module to load


module = import_module(mname)<EOL>if not module:<EOL><INDENT>raise ImportError('<STR_LIT>'.format(mname))<EOL><DEDENT>found = list()<EOL>if _should_use_module_path(module):<EOL><INDENT>mpath = module.__path__[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>mpaths = sys.path<EOL>mpath = _scan_paths_for(mname, mpaths)<EOL><DEDENT>if mpath:<EOL><INDENT>for pmname in _search_for_modules(mpath, recursive=True):<EOL><INDENT>found_mod = MODULE_PATH_SEP.join((mname, pmname))<EOL>found.append(found_mod)<EOL><DEDENT><DEDENT>return found<EOL>

Attempts to the submodules under a module recursively. This function
works for modules located in the default path as well as extended paths
via the sys.meta_path hooks.

This function carries the expectation that the hidden module variable
'__path__' has been set correctly.

:param mname: the module name to descend into


found = list()<EOL>mnames = rlist_modules(module)<EOL>for mname in mnames:<EOL><INDENT>[found.append(c) for c in list_classes(mname, cls_filter)]<EOL><DEDENT>return found<EOL>

Attempts to list all of the classes within a given module namespace.
This method, unlike list_classes, will recurse into discovered
submodules.

If a type filter is set, it will be called with each class as its
parameter. This filter's return value must be interpretable as a
boolean. Results that evaluate as True will include the type in the
list of returned classes. Results that evaluate as False will exclude
the type in the list of returned classes.

:param mname: of the module to descend into
:param cls_filter: a function to call to determine what classes should be
            included.


found = list()<EOL>if os.path.isdir(directory):<EOL><INDENT>for entry in os.listdir(directory):<EOL><INDENT>next_dir = os.path.join(directory, entry)<EOL>if os.path.isfile(os.path.join(next_dir, MODULE_INIT_FILE)):<EOL><INDENT>modules = _search_for_modules(next_dir, True, entry)<EOL>found.extend(modules)<EOL><DEDENT><DEDENT><DEDENT>return found<EOL>

Attempts to list all of the modules and submodules found within a given
directory tree. This function recursively searches the directory tree
for potential python modules and returns a list of candidate names.

**Note:** This function returns a list of strings representing
discovered module names, not the actual, loaded modules.

:param directory: the directory to search for modules.


def mapk(actual, predicted, k=<NUM_LIT:10>):

return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])<EOL>

Computes the mean average precision at k.

This function computes the mean average prescision at k between two lists
of lists of items.

Parameters
----------
actual : list
         A list of lists of elements that are to be predicted
         (order doesn't matter in the lists)
predicted : list
            A list of lists of predicted elements
            (order matters in the lists)
k : int, optional
    The maximum number of predicted elements

Returns
-------
score : double
        The mean average precision at k over the input lists


self.reg = np.zeros((self.m,), dtype=np.int8)<EOL>


if np.any(self.reg):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Returns:
    bool: True if the current HyperLogLog is empty - at the state of just
    initialized.


return type(self) is type(other) andself.seed == other.seed andnp.array_equal(self.hashvalues, other.hashvalues)<EOL>

Returns:
    bool: If their seeds and hash values are both equal then two
    are equivalent.


return WeightedMinHash(self.seed, self.digest())<EOL>

Returns:
    datasketch.WeightedMinHash: A copy of this weighted MinHash by exporting
    its state.


if other.seed != self.seed:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if len(self) != len(other):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>intersection = <NUM_LIT:0><EOL>for this, that in zip(self.hashvalues, other.hashvalues):<EOL><INDENT>if np.array_equal(this, that):<EOL><INDENT>intersection += <NUM_LIT:1><EOL><DEDENT><DEDENT>return float(intersection) / float(len(self))<EOL>

Estimate the `weighted Jaccard similarity`_ between the
        multi-sets represented by this weighted MinHash and the other.

        Args:
            other (datasketch.WeightedMinHash): The other weighted MinHash.

        Returns:
            float: The weighted Jaccard similarity between 0.0 and 1.0.

        .. _`weighted Jaccard similarity`: http://mathoverflow.net/questions/123339/weighted-jaccard-similarity


if self.prepickle:<EOL><INDENT>key = pickle.dumps(key)<EOL><DEDENT>if key not in self.keys:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>for H, hashtable in zip(self.keys[key], self.hashtables):<EOL><INDENT>hashtable.remove_val(H, key)<EOL>if not hashtable.get(H):<EOL><INDENT>hashtable.remove(H)<EOL><DEDENT><DEDENT>self.keys.remove(key)<EOL>

Remove the key from the index.

Args:
    key (hashable): The unique identifier of a set.


counts = [<EOL>hashtable.itemcounts() for hashtable in self.hashtables]<EOL>return counts<EOL>

Returns a list of length ``self.b`` with elements representing the
number of keys stored under each bucket for the given permutation.


async def get_subset_counts(self, *keys):

key_set = list(set(keys))<EOL>hashtables = [unordered_storage({'<STR_LIT:type>': '<STR_LIT>'}) for _ in range(self.b)]<EOL>Hss = await self.keys.getmany(*key_set)<EOL>for key, Hs in zip(key_set, Hss):<EOL><INDENT>for H, hashtable in zip(Hs, hashtables):<EOL><INDENT>hashtable.insert(H, key)<EOL><DEDENT><DEDENT>return [hashtable.itemcounts() for hashtable in hashtables]<EOL>


await self._remove(key, buffer=False)<EOL>


def _compute_nfp_real(l, u, counts, sizes):

if l > u:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return np.sum((float(sizes[u])-sizes[l:u+<NUM_LIT:1>])/float(sizes[u])*counts[l:u+<NUM_LIT:1>])<EOL>

Computes the expected number of false positives caused by using
    u to approximate set sizes in the interval [l, u], using the real
    set size distribution.

    Args:
        l: the lower bound on set sizes.
        u: the upper bound on set sizes.
        counts: the complete distribution of set sizes.
        sizes: the complete domain of set sizes.

    Return (float): the expected number of false positives.


nfps = np.zeros((len(sizes), len(sizes)))<EOL>for l in range(len(sizes)):<EOL><INDENT>for u in range(l, len(sizes)):<EOL><INDENT>nfps[l, u] = _compute_nfp_uniform(l, u, cum_counts, sizes)<EOL><DEDENT><DEDENT>return nfps<EOL>

Computes the matrix of expected false positives for all possible
    sub-intervals of the complete domain of set sizes, assuming uniform
    distribution of set_sizes within each sub-intervals.

    Args:
        cum_counts: the complete cummulative distribution of set sizes.
        sizes: the complete domain of set sizes.

    Return (np.array): the 2-D array of expected number of false positives
        for every pair of [l, u] interval, where l is axis-0 and u is
        axis-1.


pass<EOL>


pass<EOL>

Determines whether the key is in the storage or not


for i, index in enumerate(self.indexes):<EOL><INDENT>u = self.uppers[i]<EOL>if u is None:<EOL><INDENT>continue<EOL><DEDENT>b, r = self._get_optimal_param(u, size)<EOL>for key in index[r]._query_b(minhash, b):<EOL><INDENT>yield key<EOL><DEDENT><DEDENT>

Giving the MinHash and size of the query set, retrieve
keys that references sets with containment with respect to
the query set greater than the threshold.

Args:
    minhash (datasketch.MinHash): The MinHash of the query set.
    size (int): The size (number of unique items) of the query set.

Returns:
    `iterator` of keys.


hv = self.hashfunc(b)<EOL>a, b = self.permutations<EOL>phv = np.bitwise_and((a * hv + b) % _mersenne_prime, np.uint64(_max_hash))<EOL>self.hashvalues = np.minimum(phv, self.hashvalues)<EOL>

Update this MinHash with a new value.
        The value will be hashed using the hash function specified by
        the `hashfunc` argument in the constructor.

        Args:
            b: The value to be hashed using the hash function specified.

        Example:
            To update with a new string value (using the default SHA1 hash
            function, which requires bytes as input):

            .. code-block:: python

                minhash = Minhash()
                minhash.update("new value".encode('utf-8'))

            We can also use a different hash function, for example, `pyfarmhash`:

            .. code-block:: python

                import farmhash
                def _hash_32(b):
                    return farmhash.hash32(b)
                minhash = MinHash(hashfunc=_hash_32)
                minhash.update("new value")


return MinHash(seed=self.seed, hashfunc=self.hashfunc,<EOL>hashvalues=self.digest(),<EOL>permutations=self.permutations)<EOL>

:returns: datasketch.MinHash -- A copy of this MinHash by exporting its state.


return copy.copy(self.hashvalues)<EOL>

Export the hash values, which is the internal state of the
        MinHash.

        Returns:
            numpy.array: The hash values which is a Numpy array.


return len(self.hashvalues)<EOL>

:returns: int -- The number of hash values.


<EOL>seed_size = struct.calcsize(byteorder+'<STR_LIT:q>')<EOL>length_size = struct.calcsize(byteorder+'<STR_LIT:i>')<EOL>hashvalue_size = struct.calcsize(byteorder+'<STR_LIT:I>')<EOL>return seed_size + length_size + len(self) * hashvalue_size<EOL>

Compute the byte size after serialization.

        Args:
            byteorder (str, optional): This is byte order of the serialized data. Use one
                of the `byte order characters
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.

        Returns:
            int: Size in number of bytes after serialization.


if r == <NUM_LIT:0.0>:<EOL><INDENT>return <NUM_LIT:1.0> / (<NUM_LIT:1> << b)<EOL><DEDENT>return r * (<NUM_LIT:1> - r) ** (<NUM_LIT:2> ** b - <NUM_LIT:1>) / (<NUM_LIT:1> - (<NUM_LIT:1> - r) ** (<NUM_LIT:2> * b))<EOL>


if self.b != other.b:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if self.seed != other.seed:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>intersection = np.count_nonzero(self.hashvalues==other.hashvalues)<EOL>raw_est = float(intersection) / float(self.hashvalues.size)<EOL>a1 = self._calc_a(self.r, self.b)<EOL>a2 = self._calc_a(other.r, other.b)<EOL>c1, c2 = self._calc_c(a1, a2, self.r, other.r)<EOL>return (raw_est - c1) / (<NUM_LIT:1> - c2)<EOL>

Estimate the Jaccard similarity (resemblance) between this b-bit
MinHash and the other.


for i, hashtable in enumerate(self.hashtables):<EOL><INDENT>self.sorted_hashtables[i] = [H for H in hashtable.keys()]<EOL>self.sorted_hashtables[i].sort()<EOL><DEDENT>

Index all the keys added so far and make them searchable.


(inst, method_name, args) = arg<EOL>return getattr(inst, method_name)(*args)<EOL>

You can not call methods with multiprocessing, but free functions,
        If you want to call  inst.method(arg0, arg1),

            unwrap_self_for_multiprocessing(inst, "method", (arg0, arg1))

        does the trick.


@cli.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', required=True, type=click.Path(exists=True), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>','<STR_LIT>', type=click.Path(exists=False), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', required=True, type=click.Path(exists=True), help='<STR_LIT>')<EOL>def backpropagate(infile, outfile, apply_scores):

if outfile is None:<EOL><INDENT>outfile = infile<EOL><DEDENT>else:<EOL><INDENT>outfile = outfile<EOL><DEDENT>backpropagate_oswr(infile, outfile, apply_scores)<EOL>

Backpropagate multi-run peptide and protein scores to single files


@cli.command()<EOL>@click.argument('<STR_LIT>', nargs=-<NUM_LIT:1>, type=click.Path(exists=True))<EOL>@click.option('<STR_LIT>', '<STR_LIT>', required=True, type=click.Path(exists=True), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>def filter(sqmassfiles, infile, max_precursor_pep, max_peakgroup_pep, max_transition_pep):

filter_sqmass(sqmassfiles, infile, max_precursor_pep, max_peakgroup_pep, max_transition_pep)<EOL>


@cli.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', required=True, type=click.Path(exists=True), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>','<STR_LIT>', type=click.Path(exists=False), help='<STR_LIT>')<EOL>def reduce(infile, outfile):

if outfile is None:<EOL><INDENT>outfile = infile<EOL><DEDENT>else:<EOL><INDENT>outfile = outfile<EOL><DEDENT>reduce_osw(infile, outfile)<EOL>

Reduce scored PyProphet file to minimum for global scoring


PyProphet: Semi-supervised learning and scoring of OpenSWATH results.

Visit http://openswath.org for usage instructions and help.


@cli.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', required=True, type=click.Path(exists=True), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', '<STR_LIT>', type=click.Path(exists=False), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default='<STR_LIT>', show_default=True, type=click.Choice(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>','<STR_LIT>']), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', '<STR_LIT>', default=False, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=True, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default='<STR_LIT>', show_default=True, type=click.Choice(['<STR_LIT>','<STR_LIT>','<STR_LIT>']), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=True, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=True, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>def export(infile, outfile, format, outcsv, transition_quantification, max_transition_pep, ipf, ipf_max_peptidoform_pep, max_rs_peakgroup_qvalue, peptide, max_global_peptide_qvalue, protein, max_global_protein_qvalue):

if format == "<STR_LIT>":<EOL><INDENT>export_score_plots(infile)<EOL><DEDENT>else:<EOL><INDENT>if outfile is None:<EOL><INDENT>if outcsv:<EOL><INDENT>outfile = infile.split("<STR_LIT>")[<NUM_LIT:0>] + "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>outfile = infile.split("<STR_LIT>")[<NUM_LIT:0>] + "<STR_LIT>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>outfile = outfile<EOL><DEDENT>export_tsv(infile, outfile, format, outcsv, transition_quantification, max_transition_pep, ipf, ipf_max_peptidoform_pep, max_rs_peakgroup_qvalue, peptide, max_global_peptide_qvalue, protein, max_global_protein_qvalue)<EOL><DEDENT>


@cli.command()<EOL>@click.argument('<STR_LIT>', nargs=-<NUM_LIT:1>, type=click.Path(exists=True))<EOL>@click.option('<STR_LIT>','<STR_LIT>', required=True, type=click.Path(exists=False), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=False, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>','<STR_LIT>', required=True, type=click.Path(exists=False), help='<STR_LIT>')<EOL>def merge(infiles, outfile, same_run, templatefile):

if len(infiles) < <NUM_LIT:1>:<EOL><INDENT>raise click.ClickException("<STR_LIT>")<EOL><DEDENT>merge_osw(infiles, outfile, templatefile, same_run)<EOL>

Merge multiple OSW files and (for large experiments, it is recommended to subsample first).


@cli.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', required=True, type=click.Path(exists=True), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', '<STR_LIT>', type=click.Path(exists=False), help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=True, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=True, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=True, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=False, show_default=True, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>@click.option('<STR_LIT>', default=<NUM_LIT>, show_default=True, type=float, help='<STR_LIT>')<EOL>def ipf(infile, outfile, ipf_ms1_scoring, ipf_ms2_scoring, ipf_h0, ipf_grouped_fdr, ipf_max_precursor_pep, ipf_max_peakgroup_pep, ipf_max_precursor_peakgroup_pep, ipf_max_transition_pep):

if outfile is None:<EOL><INDENT>outfile = infile<EOL><DEDENT>else:<EOL><INDENT>outfile = outfile<EOL><DEDENT>infer_peptidoforms(infile, outfile, ipf_ms1_scoring, ipf_ms2_scoring, ipf_h0, ipf_grouped_fdr, ipf_max_precursor_pep, ipf_max_peakgroup_pep, ipf_max_precursor_peakgroup_pep, ipf_max_transition_pep)<EOL>

Infer peptidoforms after scoring of MS1, MS2 and transition-level data.


@profile<EOL>def lfdr(p_values, pi0, trunc = True, monotone = True, transf = "<STR_LIT>", adj = <NUM_LIT>, eps = np.power(<NUM_LIT>,-<NUM_LIT:8>)):

p = np.array(p_values)<EOL>lfdr_out = p<EOL>rm_na = np.isfinite(p)<EOL>p = p[rm_na]<EOL>if (min(p) < <NUM_LIT:0> or max(p) > <NUM_LIT:1>):<EOL><INDENT>raise click.ClickException("<STR_LIT>")<EOL><DEDENT>elif (pi0 < <NUM_LIT:0> or pi0 > <NUM_LIT:1>):<EOL><INDENT>raise click.ClickException("<STR_LIT>")<EOL><DEDENT>if (transf == "<STR_LIT>"):<EOL><INDENT>p = np.maximum(p, eps)<EOL>p = np.minimum(p, <NUM_LIT:1>-eps)<EOL>x = scipy.stats.norm.ppf(p, loc=<NUM_LIT:0>, scale=<NUM_LIT:1>)<EOL>bw = bw_nrd0(x)<EOL>myd = KDEUnivariate(x)<EOL>myd.fit(bw=adj*bw, gridsize = <NUM_LIT>)<EOL>splinefit = sp.interpolate.splrep(myd.support, myd.density)<EOL>y = sp.interpolate.splev(x, splinefit)<EOL>lfdr = pi0 * scipy.stats.norm.pdf(x) / y<EOL><DEDENT>elif (transf == "<STR_LIT>"):<EOL><INDENT>x = np.log((p + eps) / (<NUM_LIT:1> - p + eps))<EOL>bw = bw_nrd0(x)<EOL>myd = KDEUnivariate(x)<EOL>myd.fit(bw=adj*bw, gridsize = <NUM_LIT>)<EOL>splinefit = sp.interpolate.splrep(myd.support, myd.density)<EOL>y = sp.interpolate.splev(x, splinefit)<EOL>dx = np.exp(x) / np.power((<NUM_LIT:1> + np.exp(x)),<NUM_LIT:2>)<EOL>lfdr = (pi0 * dx) / y<EOL><DEDENT>else:<EOL><INDENT>raise click.ClickException("<STR_LIT>")<EOL><DEDENT>if (trunc):<EOL><INDENT>lfdr[lfdr > <NUM_LIT:1>] = <NUM_LIT:1><EOL><DEDENT>if (monotone):<EOL><INDENT>lfdr = lfdr[p.ravel().argsort()]<EOL>for i in range(<NUM_LIT:1>,len(x)):<EOL><INDENT>if (lfdr[i] < lfdr[i - <NUM_LIT:1>]):<EOL><INDENT>lfdr[i] = lfdr[i - <NUM_LIT:1>]<EOL><DEDENT><DEDENT>lfdr = lfdr[scipy.stats.rankdata(p,"<STR_LIT>")-<NUM_LIT:1>]<EOL><DEDENT>lfdr_out[rm_na] = lfdr<EOL>return lfdr_out<EOL>

Estimate local FDR / posterior error probability from p-values according to bioconductor/qvalue


@profile<EOL>def error_statistics(target_scores, decoy_scores, parametric, pfdr, pi0_lambda, pi0_method = "<STR_LIT>", pi0_smooth_df = <NUM_LIT:3>, pi0_smooth_log_pi0 = False, compute_lfdr = False, lfdr_trunc = True, lfdr_monotone = True, lfdr_transf = "<STR_LIT>", lfdr_adj = <NUM_LIT>, lfdr_eps = np.power(<NUM_LIT>,-<NUM_LIT:8>)):

target_scores = to_one_dim_array(target_scores)<EOL>target_scores = np.sort(target_scores[~np.isnan(target_scores)])<EOL>decoy_scores = to_one_dim_array(decoy_scores)<EOL>decoy_scores = np.sort(decoy_scores[~np.isnan(decoy_scores)])<EOL>if parametric:<EOL><INDENT>target_pvalues = pnorm(target_scores, decoy_scores)<EOL><DEDENT>else:<EOL><INDENT>target_pvalues = pemp(target_scores, decoy_scores)<EOL><DEDENT>pi0 = pi0est(target_pvalues, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0)<EOL>target_qvalues = qvalue(target_pvalues, pi0['<STR_LIT>'], pfdr)<EOL>metrics = stat_metrics(target_pvalues, pi0['<STR_LIT>'], pfdr)<EOL>error_stat = pd.DataFrame({'<STR_LIT>': target_scores, '<STR_LIT>': target_pvalues, '<STR_LIT>': target_qvalues, '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>'], '<STR_LIT>': metrics['<STR_LIT>']})<EOL>if compute_lfdr:<EOL><INDENT>error_stat['<STR_LIT>'] = lfdr(target_pvalues, pi0['<STR_LIT>'], lfdr_trunc, lfdr_monotone, lfdr_transf, lfdr_adj, lfdr_eps)<EOL><DEDENT>return error_stat, pi0<EOL>

Takes list of decoy and target scores and creates error statistics for target values


mu, sigma = mean_and_std_dev(stat0)<EOL>stat = to_one_dim_array(stat, np.float64)<EOL>args = (stat - mu) / sigma<EOL>return <NUM_LIT:1>-(<NUM_LIT:0.5> * (<NUM_LIT:1.0> + scipy.special.erf(args / np.sqrt(<NUM_LIT>))))<EOL>

[P(X>pi, mu, sigma) for pi in pvalues] for normal distributed stat with
    expectation value mu and std deviation sigma


@profile<EOL>def summary_err_table(df, qvalues=[<NUM_LIT:0>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT:0.1>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT:0.5>]):

qvalues = to_one_dim_array(qvalues)<EOL>ix = find_nearest_matches(np.float32(df.qvalue.values), qvalues)<EOL>df_sub = df.iloc[ix].copy()<EOL>for i_sub, (i0, i1) in enumerate(zip(ix, ix[<NUM_LIT:1>:])):<EOL><INDENT>if i1 == i0:<EOL><INDENT>df_sub.iloc[i_sub + <NUM_LIT:1>, :] = None<EOL><DEDENT><DEDENT>df_sub.qvalue = qvalues<EOL>df_sub.reset_index(inplace=True, drop=True)<EOL>return df_sub[['<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>']]<EOL>

Summary error table for some typical q-values


def find_cutoff(tt_scores, td_scores, cutoff_fdr, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0):

error_stat, pi0 = error_statistics(tt_scores, td_scores, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, False)<EOL>if not len(error_stat):<EOL><INDENT>raise click.ClickException("<STR_LIT>")<EOL><DEDENT>i0 = (error_stat.qvalue - cutoff_fdr).abs().idxmin()<EOL>cutoff = error_stat.iloc[i0]["<STR_LIT>"]<EOL>return cutoff<EOL>

Finds cut off target score for specified false discovery rate fdr


if not settings.UNFRIENDLY_ENABLE_FILTER:<EOL><INDENT>return value<EOL><DEDENT>kwargs = {<EOL>'<STR_LIT:key>': encrypt(value,<EOL>settings.UNFRIENDLY_SECRET,<EOL>settings.UNFRIENDLY_IV,<EOL>checksum=settings.UNFRIENDLY_ENFORCE_CHECKSUM),<EOL>}<EOL>if juice:<EOL><INDENT>kwargs['<STR_LIT>'] = slugify(juice)<EOL><DEDENT>return reverse('<STR_LIT>', kwargs=kwargs)<EOL>

Template filter that obfuscates whatever text it is applied to. The text is
supposed to be a URL, but it will obfuscate anything.

Usage:
    Extremely unfriendly URL:
    {{ "/my-secret-path/"|obfuscate }}

    Include some SEO juice:
    {{ "/my-secret-path/"|obfuscate:"some SEO friendly text" }}


def decrypt(ciphertext, secret, inital_vector, checksum=True, lazy=True):

secret = _lazysecret(secret) if lazy else secret<EOL>encobj = AES.new(secret, AES.MODE_CFB, inital_vector)<EOL>try:<EOL><INDENT>padded = ciphertext + ('<STR_LIT:=>' * (len(ciphertext) % <NUM_LIT:4>))<EOL>decoded = base64.urlsafe_b64decode(str(padded))<EOL>plaintext = encobj.decrypt(decoded)<EOL><DEDENT>except (TypeError, binascii.Error):<EOL><INDENT>raise InvalidKeyError("<STR_LIT>")<EOL><DEDENT>if checksum:<EOL><INDENT>try:<EOL><INDENT>crc, plaintext = (base64.urlsafe_b64decode(<EOL>plaintext[-<NUM_LIT:8>:]), plaintext[:-<NUM_LIT:8>])<EOL><DEDENT>except (TypeError, binascii.Error):<EOL><INDENT>raise CheckSumError("<STR_LIT>")<EOL><DEDENT>if not crc == _pack_crc(plaintext):<EOL><INDENT>raise CheckSumError("<STR_LIT>")<EOL><DEDENT><DEDENT>return plaintext<EOL>

Decrypts ciphertext with secret
    ciphertext     - encrypted content to decrypt
    secret         - secret to decrypt ciphertext
    inital_vector  - initial vector
    lazy           - pad secret if less than legal blocksize (default: True)
    checksum       - verify crc32 byte encoded checksum (default: True)
    returns plaintext


if not isinstance(plaintext, six.binary_type):<EOL><INDENT>plaintext = six.b(plaintext)<EOL><DEDENT>return (zlib.crc32(plaintext) % <NUM_LIT>) & <NUM_LIT><EOL>

Generates crc32. Modulo keep the value within int range.


@njit()<EOL>def _cluster_hits(hits, clusters, assigned_hit_array, cluster_hit_indices, column_cluster_distance, row_cluster_distance, frame_cluster_distance, min_hit_charge, max_hit_charge, ignore_same_hits, noisy_pixels, disabled_pixels):

total_hits = hits.shape[<NUM_LIT:0>]<EOL>if total_hits == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0>  <EOL><DEDENT>max_cluster_hits = cluster_hit_indices.shape[<NUM_LIT:0>]<EOL>if total_hits != clusters.shape[<NUM_LIT:0>]:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if total_hits != assigned_hit_array.shape[<NUM_LIT:0>]:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if min_hit_charge == <NUM_LIT:0>:<EOL><INDENT>charge_correction = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>charge_correction = <NUM_LIT:0><EOL><DEDENT>start_event_hit_index = <NUM_LIT:0><EOL>start_event_cluster_index = <NUM_LIT:0><EOL>cluster_size = <NUM_LIT:0><EOL>event_number = hits[<NUM_LIT:0>]['<STR_LIT>']<EOL>event_cluster_index = <NUM_LIT:0><EOL>for i in range(total_hits):<EOL><INDENT>if _new_event(hits[i]['<STR_LIT>'], event_number):<EOL><INDENT>_finish_event(<EOL>hits=hits,<EOL>clusters=clusters,<EOL>start_event_hit_index=start_event_hit_index,<EOL>stop_event_hit_index=i,<EOL>start_event_cluster_index=start_event_cluster_index,<EOL>stop_event_cluster_index=start_event_cluster_index + event_cluster_index)<EOL>start_event_hit_index = i<EOL>start_event_cluster_index = start_event_cluster_index + event_cluster_index<EOL>event_number = hits[i]['<STR_LIT>']<EOL>event_cluster_index = <NUM_LIT:0><EOL><DEDENT>if assigned_hit_array[i] > <NUM_LIT:0>:  <EOL><INDENT>continue<EOL><DEDENT>if not _hit_ok(<EOL>hit=hits[i],<EOL>min_hit_charge=min_hit_charge,<EOL>max_hit_charge=max_hit_charge) or (disabled_pixels.shape[<NUM_LIT:0>] != <NUM_LIT:0> and _pixel_masked(hits[i], disabled_pixels)):<EOL><INDENT>_set_hit_invalid(hit=hits[i], cluster_id=-<NUM_LIT:1>)<EOL>assigned_hit_array[i] = <NUM_LIT:1><EOL>continue<EOL><DEDENT>_set_1d_array(cluster_hit_indices, -<NUM_LIT:1>, cluster_size)<EOL>cluster_hit_indices[<NUM_LIT:0>] = i<EOL>assigned_hit_array[i] = <NUM_LIT:1><EOL>cluster_size = <NUM_LIT:1>  <EOL>for j in cluster_hit_indices:  <EOL><INDENT>if j < <NUM_LIT:0>:  <EOL><INDENT>break<EOL><DEDENT>for k in range(cluster_hit_indices[<NUM_LIT:0>] + <NUM_LIT:1>, total_hits):<EOL><INDENT>if _new_event(hits[k]['<STR_LIT>'], event_number):<EOL><INDENT>break<EOL><DEDENT>if assigned_hit_array[k] > <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>if not _hit_ok(<EOL>hit=hits[k],<EOL>min_hit_charge=min_hit_charge,<EOL>max_hit_charge=max_hit_charge) or (disabled_pixels.shape[<NUM_LIT:0>] != <NUM_LIT:0> and _pixel_masked(hits[k], disabled_pixels)):<EOL><INDENT>_set_hit_invalid(hit=hits[k], cluster_id=-<NUM_LIT:1>)<EOL>assigned_hit_array[k] = <NUM_LIT:1><EOL>continue<EOL><DEDENT>if _is_in_max_difference(hits[j]['<STR_LIT>'], hits[k]['<STR_LIT>'], column_cluster_distance) and _is_in_max_difference(hits[j]['<STR_LIT>'], hits[k]['<STR_LIT>'], row_cluster_distance) and _is_in_max_difference(hits[j]['<STR_LIT>'], hits[k]['<STR_LIT>'], frame_cluster_distance):<EOL><INDENT>if not ignore_same_hits or hits[j]['<STR_LIT>'] != hits[k]['<STR_LIT>'] or hits[j]['<STR_LIT>'] != hits[k]['<STR_LIT>']:<EOL><INDENT>cluster_size += <NUM_LIT:1><EOL>if cluster_size > max_cluster_hits:<EOL><INDENT>raise IndexError('<STR_LIT>')<EOL><DEDENT>cluster_hit_indices[cluster_size - <NUM_LIT:1>] = k<EOL>assigned_hit_array[k] = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>_set_hit_invalid(hit=hits[k], cluster_id=-<NUM_LIT:2>)<EOL>assigned_hit_array[k] = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>if cluster_size == <NUM_LIT:1> and noisy_pixels.shape[<NUM_LIT:0>] != <NUM_LIT:0> and _pixel_masked(hits[cluster_hit_indices[<NUM_LIT:0>]], noisy_pixels):<EOL><INDENT>_set_hit_invalid(hit=hits[cluster_hit_indices[<NUM_LIT:0>]], cluster_id=-<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>_finish_cluster(<EOL>hits=hits,<EOL>clusters=clusters,<EOL>cluster_size=cluster_size,<EOL>cluster_hit_indices=cluster_hit_indices,<EOL>cluster_index=start_event_cluster_index + event_cluster_index,<EOL>cluster_id=event_cluster_index,<EOL>charge_correction=charge_correction,<EOL>noisy_pixels=noisy_pixels,<EOL>disabled_pixels=disabled_pixels)<EOL>event_cluster_index += <NUM_LIT:1><EOL><DEDENT><DEDENT>_finish_event(<EOL>hits=hits,<EOL>clusters=clusters,<EOL>start_event_hit_index=start_event_hit_index,<EOL>stop_event_hit_index=total_hits,<EOL>start_event_cluster_index=start_event_cluster_index,<EOL>stop_event_cluster_index=start_event_cluster_index + event_cluster_index)<EOL>total_clusters = start_event_cluster_index + event_cluster_index<EOL>return total_clusters<EOL>

Main precompiled function that loopes over the hits and clusters them


@njit()<EOL>def _is_in_max_difference(value_1, value_2, max_difference):

if value_1 <= value_2:<EOL><INDENT>return value_2 - value_1 <= max_difference<EOL><DEDENT>return value_1 - value_2 <= max_difference<EOL>

Helper function to determine the difference of two values that can be np.uints. Works in python and numba mode.
    Circumvents numba bug #1653