def __init__(self, logic=None, weight=None, simple=None, simple_weight=None, extract_as=None, extract_all=None,<EOL>extract_when_missing=None, tags=None, length=None, offset=None, number=None, title=None,<EOL>caption=None, query=None, **kwargs):

super(DisplayItemQuery, self).__init__(<EOL>logic=logic, weight=weight, simple=simple, simple_weight=simple_weight, extract_as=extract_as,<EOL>extract_all=extract_all, extract_when_missing=extract_when_missing, tags=tags, length=length,<EOL>offset=offset, **kwargs)<EOL>self._title = None<EOL>self.title = title<EOL>self._number = None<EOL>self.number = number<EOL>self._caption = None<EOL>self.caption = caption<EOL>self._query = None<EOL>self.query = query<EOL>

Constructor.

:param logic: Logic for this filter. Must be equal to one of "MUST", "MUST_NOT", "SHOULD", or "OPTIONAL".
:param weight: Weight of the query.
:param simple: String with the simple query to run against all fields.
:param simple_weight: Dictionary of relative paths to their weights for simple queries.
:param extract_as: String with the alias to save this field under.
:param extract_all: Boolean setting whether all values in an array should be extracted.
:param extract_when_missing: Any valid JSON-supported object or PIF object. This value is returned when a value is missing that should be extracted (and the overall query is still satisfied).
:param tags: One or more :class:`FieldQuery` operations against the tags field.
:param length: One or more :class:`FieldQuery` operations against the length field.
:param offset: One or more :class:`FieldQuery` operations against the offset field.
:param number: One or more :class:`FieldQuery` operations against the number field.
:param title: One or more :class:`FieldQuery` operations against the title field.
:param caption: One or more :class:`FieldQuery` operations against the caption field.
:param query: One or more :class:`DisplayItemQuery` objects as nested queries.


def __init__(self, logic=None, weight=None, simple=None, simple_weight=None, extract_as=None, extract_all=None,<EOL>extract_when_missing=None, tags=None, length=None, offset=None, producer=None, url=None,<EOL>query=None, **kwargs):

super(SourceQuery, self).__init__(<EOL>logic=logic, weight=weight, simple=simple, simple_weight=simple_weight, extract_as=extract_as,<EOL>extract_all=extract_all, extract_when_missing=extract_when_missing, tags=tags, length=length,<EOL>offset=offset, **kwargs)<EOL>self._producer = None<EOL>self.producer = producer<EOL>self._url = None<EOL>self.url = url<EOL>self._query = None<EOL>self.query = query<EOL>

Constructor.

:param logic: Logic for this filter. Must be equal to one of "MUST", "MUST_NOT", "SHOULD", or "OPTIONAL".
:param weight: Weight of the query.
:param simple: String with the simple query to run against all fields.
:param simple_weight: Dictionary of relative paths to their weights for simple queries.
:param extract_as: String with the alias to save this field under.
:param extract_all: Boolean setting whether all values in an array should be extracted.
:param extract_when_missing: Any valid JSON-supported object or PIF object. This value is returned when a value is missing that should be extracted (and the overall query is still satisfied).
:param tags: One or more :class:`FieldQuery` operations against the tags field.
:param length: One or more :class:`FieldQuery` operations against the length field.
:param offset: One or more :class:`FieldQuery` operations against the offset field.
:param producer: One or more :class:`FieldQuery` operations against the producer field.
:param url: One or more :class:`FieldQuery` operations against the url field.
:param query: One or more :class:`SourceQuery` objects with nested queries.


def __init__(self, logic=None, weight=None, simple=None, simple_weight=None, extract_as=None, extract_all=None,<EOL>extract_when_missing=None, tags=None, length=None, offset=None, doi=None, isbn=None, issn=None,<EOL>url=None, title=None, publisher=None, journal=None, volume=None, issue=None, year=None,<EOL>figure=None, table=None, pages=None, authors=None, editors=None, affiliations=None,<EOL>acknowledgements=None, references=None, query=None, **kwargs):

super(ReferenceQuery, self).__init__(<EOL>logic=logic, weight=weight, simple=simple, simple_weight=simple_weight, extract_as=extract_as,<EOL>extract_all=extract_all, extract_when_missing=extract_when_missing, tags=tags, length=length,<EOL>offset=offset, **kwargs)<EOL>self._doi = None<EOL>self.doi = doi<EOL>self._isbn = None<EOL>self.isbn = isbn<EOL>self._issn = None<EOL>self.issn = issn<EOL>self._url = None<EOL>self.url = url<EOL>self._title = None<EOL>self.title = title<EOL>self._publisher = None<EOL>self.publisher = publisher<EOL>self._journal = None<EOL>self.journal = journal<EOL>self._volume = None<EOL>self.volume = volume<EOL>self._issue = None<EOL>self.issue = issue<EOL>self._year = None<EOL>self.year = year<EOL>self._figure = None<EOL>self.figure = figure<EOL>self._table = None<EOL>self.table = table<EOL>self._pages = None<EOL>self.pages = pages<EOL>self._authors = None<EOL>self.authors = authors<EOL>self._editors = None<EOL>self.editors = editors<EOL>self._affiliations = None<EOL>self.affiliations = affiliations<EOL>self._acknowledgements = None<EOL>self.acknowledgements = acknowledgements<EOL>self._references = None<EOL>self.references = references<EOL>self._query = None<EOL>self.query = query<EOL>

Constructor.

:param logic: Logic for this filter. Must be equal to one of "MUST", "MUST_NOT", "SHOULD", or "OPTIONAL".
:param weight: Weight of the query.
:param simple: String with the simple query to run against all fields.
:param simple_weight: Dictionary of relative paths to their weights for simple queries.
:param extract_as: String with the alias to save this field under.
:param extract_all: Boolean setting whether all values in an array should be extracted.
:param extract_when_missing: Any valid JSON-supported object or PIF object. This value is returned when a value is missing that should be extracted (and the overall query is still satisfied).
:param tags: One or more :class:`FieldQuery` operations against the tags field.
:param length: One or more :class:`FieldQuery` operations against the length field.
:param offset: One or more :class:`FieldQuery` operations against the offset field.
:param doi: One or more :class:`FieldQuery` operations against the doi field.
:param isbn: One or more :class:`FieldQuery` operations against the isbn field.
:param issn: One or more :class:`FieldQuery` operations against the issn field.
:param url: One or more :class:`FieldQuery` operations against the url field.
:param title: One or more :class:`FieldQuery` operations against the title field.
:param publisher: One or more :class:`FieldQuery` operations against the publisher field.
:param journal: One or more :class:`FieldQuery` operations against the journal field.
:param volume: One or more :class:`FieldQuery` operations against the volume field.
:param issue: One or more :class:`FieldQuery` operations against the issue field.
:param year: One or more :class:`FieldQuery` operations against the year field.
:param figure: One or more :class:`DisplayItemQuery` operations against the figure field.
:param table: One or more :class:`DisplayItemQuery` operations against the table field.
:param pages: One or more :class:`PagesQuery` operations against the pages field.
:param authors: One or more :class:`NameQuery` operations against the authors field.
:param editors: One or more :class:`NameQuery` operations against the editors field.
:param affiliations: One or more :class:`FieldQuery` operations against the affiliations field.
:param acknowledgements: One or more :class:`FieldQuery` operations against the acknowledgements field.
:param references: One or more :class:`ReferenceQuery` operations against the references field.
:param query: One or more :class:`ReferenceQuery` objects with nested queries.


def __init__(self, sort=None, weight=None, logic=None, simple=None, simple_weight=None, extract_as=None,<EOL>extract_all=None, extract_when_missing=None, length=None, offset=None, filter=None, **kwargs):

super(FieldQuery, self).__init__(<EOL>sort=sort, weight=weight, logic=logic, simple=simple, simple_weight=simple_weight, extract_as=extract_as,<EOL>extract_all=extract_all, extract_when_missing=extract_when_missing, length=length, offset=offset, **kwargs)<EOL>self._filter = None<EOL>self.filter = filter<EOL>

Constructor.

:param sort: ASCENDING or DESCENDING to set the sort order on this field.
:param weight: Weight of the query.
:param logic: Logic for this query. Must be equal to one of "MUST", "MUST_NOT", "SHOULD", or "OPTIONAL".
:param simple: String with the simple search to run against all fields.
:param simple_weight: Dictionary of relative paths to their weights for simple queries.
:param extract_as: String with the alias to save this field under.
:param extract_all: Boolean setting whether all values in an array should be extracted.
:param extract_when_missing: Any valid JSON-supported object or PIF object. This value is returned when a value is missing that should be extracted (and the overall query is still satisfied).
:param length: One or more :class:`FieldQuery` objects against the length field.
:param offset: One or more :class:`FieldQuery` objects against the offset field.
:param filter: One or more :class:`Filter` objects against this field.


def __init__(self, logic=None, weight=None, simple=None, simple_weight=None, extract_as=None, extract_all=None,<EOL>extract_when_missing=None, tags=None, length=None, offset=None, **kwargs):

self._logic = None<EOL>self.logic = logic<EOL>self._weight = None<EOL>self.weight = weight<EOL>self._simple = None<EOL>self.simple = simple<EOL>self._simple_weight = None<EOL>self.simple_weight = simple_weight<EOL>self._extract_as = None<EOL>self.extract_as = extract_as<EOL>self._extract_all = None<EOL>self.extract_all = extract_all<EOL>self._extract_when_missing = None<EOL>self.extract_when_missing = extract_when_missing<EOL>self._tags = None<EOL>self.tags = tags<EOL>self._length = None<EOL>self.length = length<EOL>self._offset = None<EOL>self.offset = offset<EOL>

Constructor.

:param logic: Logic for this filter. Must be equal to one of "MUST", "MUST_NOT", "SHOULD", or "OPTIONAL".
:param weight: Weight for the query.
:param simple: String with the simple search to run against all fields.
:param simple_weight: Dictionary of relative paths to their weights for simple queries.
:param extract_as: String with the alias to save this field under.
:param extract_all: Boolean setting whether all values in an array should be extracted.
:param extract_when_missing: Any valid JSON-supported object or PIF object. This value is returned when a value is missing that should be extracted (and the overall query is still satisfied).
:param tags: One or more :class:`FieldQuery` operations against the tags field.
:param length: One or more :class:`FieldQuery` operations against the length field.
:param offset: One or more :class:`FieldQuery` operations against the offset field.


def __init__(self, id=None, dataset=None, dataset_version=None, score=None, updated_at=None, system=None, <EOL>extracted=None, extracted_path=None, **kwargs):

self._id = None<EOL>self.id = id<EOL>self._dataset = None<EOL>self.dataset = dataset<EOL>self._dataset_version = None<EOL>self.dataset_version = dataset_version<EOL>self._score = None<EOL>self.score = score<EOL>self._updated_at = None<EOL>self.updated_at = updated_at<EOL>self._system = None<EOL>self.system = system<EOL>self._extracted = None<EOL>self.extracted = extracted<EOL>self._extracted_path = None<EOL>self.extracted_path = extracted_path<EOL>

Constructor.

:param id: String with the ID of the record.
:param dataset: Integer with the dataset of the record.
:param dataset_version: Integer with the dataset version of the record.
:param score: Score with the relevancy of the result.
:param updated_at: String with the last time that the record was updated.
:param system: Pif System object that matched.
:param extracted: Dictionary with a map of extracted property names to values.
:param extracted_path: Dictionary with a map of extracted property names to paths in a PIF.


with open(path, "<STR_LIT:r>") as f:<EOL><INDENT>parsed_dict = json.load(f)<EOL><DEDENT>return parsed_dict<EOL>

Given a filepath, loads the file as a dictionary from JSON

:param path: The path to a JSON file


failure_message = "<STR_LIT>"<EOL>response = self._get_success_json(self._get(<EOL>'<STR_LIT>' + job_id + '<STR_LIT>', None, failure_message=failure_message))[<EOL>'<STR_LIT:data>']<EOL>return response<EOL>

After invoking the create_ml_configuration async method, you can use this method to
check on the status of the builder job.

:param job_id: The identifier returned from create_ml_configuration
:return: Job status


def create_ml_configuration(self, search_template, extract_as_keys, dataset_ids):

data = {<EOL>"<STR_LIT>":<EOL>search_template,<EOL>"<STR_LIT>":<EOL>extract_as_keys<EOL>}<EOL>failure_message = "<STR_LIT>"<EOL>config_job_id = self._get_success_json(self._post_json(<EOL>'<STR_LIT>', data, failure_message=failure_message))['<STR_LIT:data>'][<EOL>'<STR_LIT:result>']['<STR_LIT>']<EOL>while True:<EOL><INDENT>config_status = self.__get_ml_configuration_status(config_job_id)<EOL>print('<STR_LIT>', config_status)<EOL>if config_status['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>ml_config = self.__convert_response_to_configuration(config_status['<STR_LIT:result>'], dataset_ids)<EOL>return ml_config<EOL><DEDENT>time.sleep(<NUM_LIT:5>)<EOL><DEDENT>

This method will spawn a server job to create a default ML configuration based on a search template and
the extract as keys.
This function will submit the request to build, and wait for the configuration to finish before returning.

:param search_template: A search template defining the query (properties, datasets etc)
:param extract_as_keys: Array of extract-as keys defining the descriptors
:param dataset_ids: Array of dataset identifiers to make search template from
:return: An identifier used to request the status of the builder job (get_ml_configuration_status)


available_columns = self.search_template_client.get_available_columns(dataset_ids)<EOL>search_template = self.search_template_client.create(dataset_ids, available_columns)<EOL>return self.create_ml_configuration(search_template, available_columns, dataset_ids)<EOL>

Creates an ml configuration from dataset_ids and extract_as_keys

:param dataset_ids: Array of dataset identifiers to make search template from
:return: An identifier used to request the status of the builder job (get_ml_configuration_status)


data = {<EOL>"<STR_LIT>":<EOL>ml_template<EOL>}<EOL>failure_message = "<STR_LIT>"<EOL>res = self._get_success_json(self._post_json(<EOL>'<STR_LIT>', data, failure_message=failure_message))['<STR_LIT:data>']<EOL>if res['<STR_LIT>']:<EOL><INDENT>return '<STR_LIT:OK>'<EOL><DEDENT>return res['<STR_LIT>']<EOL>

Runs the template against the validation endpoint, returns a message indicating status of the templte

:param ml_template: Template to validate
:return: OK or error message if validation failed


def add_descriptor(self, descriptor, role='<STR_LIT:ignore>', group_by_key=False):

descriptor.validate()<EOL>if descriptor.key in self.configuration["<STR_LIT>"]:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.configuration['<STR_LIT>'].append(descriptor.as_dict())<EOL>self.configuration["<STR_LIT>"][descriptor.key] = role<EOL>if group_by_key:<EOL><INDENT>self.configuration["<STR_LIT>"].append(descriptor.key)<EOL><DEDENT>

Add a descriptor column.

:param descriptor: A Descriptor instance (e.g., RealDescriptor, InorganicDescriptor, etc.)
:param role: Specify a role (input, output, latentVariable, or ignore)
:param group_by_key: Whether or not to group by this key during cross validation


self.configuration['<STR_LIT>'] = dataset_ids<EOL>

Sets the dataset ids to use for the view

:param dataset_ids: Array of strings, one for each dataset id


if not isinstance(dataset_ids, list):<EOL><INDENT>dataset_ids = [dataset_ids]<EOL><DEDENT>data = {<EOL>"<STR_LIT>":<EOL>dataset_ids<EOL>}<EOL>failure_message = "<STR_LIT>".format(dataset_ids)<EOL>return self._get_success_json(self._post_json(<EOL>'<STR_LIT>', data, failure_message=failure_message))['<STR_LIT:data>']<EOL>

Retrieves the set of columns from the combination of dataset ids given

:param dataset_ids: The id of the dataset to retrieve columns from
:type dataset_ids: list of int
:return: A list of column names from the dataset ids given.
:rtype: list of str


return self._ip_dec<EOL>

Return the decimal representation of the address/netmask.


if check and not is_dot(ip):<EOL><INDENT>raise ValueError('<STR_LIT>' % ip)<EOL><DEDENT>octets = str(ip).split('<STR_LIT:.>')<EOL>dec = <NUM_LIT:0><EOL>dec |= int(octets[<NUM_LIT:0>]) << <NUM_LIT><EOL>dec |= int(octets[<NUM_LIT:1>]) << <NUM_LIT:16><EOL>dec |= int(octets[<NUM_LIT:2>]) << <NUM_LIT:8><EOL>dec |= int(octets[<NUM_LIT:3>])<EOL>return dec<EOL>

Dotted decimal notation to decimal conversion.


return self._net_ip<EOL>


<EOL>_NM_CHECK_FUNCT = {<EOL>NM_DOT: _dot_to_dec,<EOL>NM_HEX: _hex_to_dec,<EOL>NM_BIN: _bin_to_dec,<EOL>NM_OCT: _oct_to_dec,<EOL>NM_DEC: _dec_to_dec_long}<EOL>try:<EOL><INDENT>dec = _NM_CHECK_FUNCT[notation](nm, check=True)<EOL><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>if dec in _NETMASKS_VALUES:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

Function internally used to check if the given netmask
    is of the specified notation.


self.set(ip=ip, netmask=self._nm)<EOL>


return _check_nm(nm, NM_HEX)<EOL>

Return true if the netmask is in hexadecimal notatation.


return _check_nm(nm, NM_DEC)<EOL>

Return true if the netmask is in decimal notatation.


try:<EOL><INDENT>dec = int(str(ip), <NUM_LIT:16>)<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>return False<EOL><DEDENT>if dec > <NUM_LIT> or dec < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Return true if the IP address is in hexadecimal notation.


return _detect(ip, _isnm=False)<EOL>

Detect the notation of an IP address.

    @param ip: the IP address.
    @type ip: integers, strings or object with an appropriate __str()__ method.
    @return: one of the IP_* constants; IP_UNKNOWN if undetected.


return NOTATION_MAP[detect(ip)][<NUM_LIT:0>]<EOL>

Return the notation of an IP address (string).


return hex(ip)<EOL>


return _check_nm(nm, NM_BIN)<EOL>

Return true if the netmask is in binary notatation.


self.set(self._add(other), notation=IP_DEC)<EOL>return self<EOL>


return oct(ip)<EOL>


return _detect(nm, _isnm=True)<EOL>

Detect the notation of a netmask.
    @param nm: the netmask.
    @type nm: integers, strings or object with an appropriate __str()__ method.
    @return: one of the NM_* constants; NM_UNKNOWN if undetected.


octets = str(ip).split('<STR_LIT:.>')<EOL>if len(octets) != <NUM_LIT:4>:<EOL><INDENT>return False<EOL><DEDENT>for i in octets:<EOL><INDENT>try:<EOL><INDENT>val = int(i)<EOL><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>if val > <NUM_LIT:255> or val < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Return true if the IP address is in dotted decimal notation.


try:<EOL><INDENT>dec = int(str(ip))<EOL><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>if dec > <NUM_LIT> or dec < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Return true if the IP address is in decimal notation.


if check and not is_hex(ip):<EOL><INDENT>raise ValueError('<STR_LIT>' % ip)<EOL><DEDENT>if isinstance(ip, int):<EOL><INDENT>ip = hex(ip)<EOL><DEDENT>return int(str(ip), <NUM_LIT:16>)<EOL>


if isinstance(other, self.__class__):<EOL><INDENT>return other._ip_dec<EOL><DEDENT>elif isinstance(other, int):<EOL><INDENT>return other<EOL><DEDENT>return self.__class__(other)._ip_dec<EOL>

Prepare the item to be compared with this address/netmask.


return self._ip<EOL>


return _is_notation(ip, notation, _isnm=False)<EOL>

Return true if the given address is in the given notation.


return self._ip<EOL>

Return the dotted decimal notation of the address/netmask.


try:<EOL><INDENT>dec = int(str(ip), <NUM_LIT:8>)<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>return False<EOL><DEDENT>if dec > <NUM_LIT> or dec < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Return true if the IP address is in octal notation.


if check and not is_wildcard_nm(nm):<EOL><INDENT>raise ValueError('<STR_LIT>' % nm)<EOL><DEDENT>return <NUM_LIT> - _dot_to_dec(nm, check=False)<EOL>


def __init__(self, ip, notation=IP_UNKNOWN):

self.set(ip, notation)<EOL>


if isinstance(ip, str) and netmask is None:<EOL><INDENT>ipnm = ip.split('<STR_LIT:/>')<EOL>if len(ipnm) != <NUM_LIT:2>:<EOL><INDENT>raise ValueError('<STR_LIT>' % ip)<EOL><DEDENT>ip = ipnm[<NUM_LIT:0>]<EOL>netmask = ipnm[<NUM_LIT:1>]<EOL><DEDENT>if isinstance(ip, IPv4Address):<EOL><INDENT>self._ip = ip<EOL><DEDENT>else:<EOL><INDENT>self._ip = IPv4Address(ip)<EOL><DEDENT>if isinstance(netmask, IPv4NetMask):<EOL><INDENT>self._nm = netmask<EOL><DEDENT>else:<EOL><INDENT>self._nm = IPv4NetMask(netmask)<EOL><DEDENT>ipl = int(self._ip)<EOL>nml = int(self._nm)<EOL>base_add = ipl & nml<EOL>self._ip_num = <NUM_LIT> - <NUM_LIT:1> - nml<EOL>if self._ip_num in (-<NUM_LIT:1>, <NUM_LIT:0>):<EOL><INDENT>if self._ip_num == -<NUM_LIT:1>:<EOL><INDENT>self._ip_num = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self._ip_num = <NUM_LIT:2><EOL><DEDENT>self._net_ip = None<EOL>self._bc_ip = None<EOL>self._first_ip_dec = base_add<EOL>self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)<EOL>if self._ip_num == <NUM_LIT:1>:<EOL><INDENT>last_ip_dec = self._first_ip_dec<EOL><DEDENT>else:<EOL><INDENT>last_ip_dec = self._first_ip_dec + <NUM_LIT:1><EOL><DEDENT>self._last_ip = IPv4Address(last_ip_dec, notation=IP_DEC)<EOL>return<EOL><DEDENT>self._net_ip = IPv4Address(base_add, notation=IP_DEC)<EOL>self._bc_ip = IPv4Address(base_add + self._ip_num + <NUM_LIT:1>, notation=IP_DEC)<EOL>self._first_ip_dec = base_add + <NUM_LIT:1><EOL>self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)<EOL>self._last_ip = IPv4Address(base_add + self._ip_num, notation=IP_DEC)<EOL>


return _convert(self._ip, notation=NM_WILDCARD,<EOL>inotation=IP_DOT, _check=False, _isnm=self._isnm)<EOL>

Return the wildcard bits notation of the netmask.


return NOTATION_MAP[detect_nm(nm)][<NUM_LIT:0>]<EOL>

Return the notation of a netmask (string).


first = int((ip >> <NUM_LIT>) & <NUM_LIT:255>)<EOL>second = int((ip >> <NUM_LIT:16>) & <NUM_LIT:255>)<EOL>third = int((ip >> <NUM_LIT:8>) & <NUM_LIT:255>)<EOL>fourth = int(ip & <NUM_LIT:255>)<EOL>return '<STR_LIT>' % (first, second, third, fourth)<EOL>

Decimal to dotted decimal notation conversion.


def _convert(ip, notation, inotation, _check, _isnm):

inotation_orig = inotation<EOL>notation_orig = notation<EOL>inotation = _get_notation(inotation)<EOL>notation = _get_notation(notation)<EOL>if inotation is None:<EOL><INDENT>raise ValueError('<STR_LIT>' % inotation_orig)<EOL><DEDENT>if notation is None:<EOL><INDENT>raise ValueError('<STR_LIT>' % notation_orig)<EOL><DEDENT>docheck = _check or False<EOL>if inotation == IP_UNKNOWN:<EOL><INDENT>inotation = _detect(ip, _isnm)<EOL>if inotation == IP_UNKNOWN:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if _check is None:<EOL><INDENT>docheck = True<EOL><DEDENT><DEDENT>if _isnm:<EOL><INDENT>docheck = False<EOL><DEDENT>dec = <NUM_LIT:0><EOL>if inotation == IP_DOT:<EOL><INDENT>dec = _dot_to_dec(ip, docheck)<EOL><DEDENT>elif inotation == IP_HEX:<EOL><INDENT>dec = _hex_to_dec(ip, docheck)<EOL><DEDENT>elif inotation == IP_BIN:<EOL><INDENT>dec = _bin_to_dec(ip, docheck)<EOL><DEDENT>elif inotation == IP_OCT:<EOL><INDENT>dec = _oct_to_dec(ip, docheck)<EOL><DEDENT>elif inotation == IP_DEC:<EOL><INDENT>dec = _dec_to_dec_long(ip, docheck)<EOL><DEDENT>elif _isnm and inotation == NM_BITS:<EOL><INDENT>dec = _bits_to_dec(ip, docheck)<EOL><DEDENT>elif _isnm and inotation == NM_WILDCARD:<EOL><INDENT>dec = _wildcard_to_dec(ip, docheck)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % inotation_orig)<EOL><DEDENT>if _isnm and dec not in _NETMASKS_VALUES:<EOL><INDENT>raise ValueError('<STR_LIT>' % ip)<EOL><DEDENT>if notation == IP_DOT:<EOL><INDENT>return _dec_to_dot(dec)<EOL><DEDENT>elif notation == IP_HEX:<EOL><INDENT>return _dec_to_hex(dec)<EOL><DEDENT>elif notation == IP_BIN:<EOL><INDENT>return _dec_to_bin(dec)<EOL><DEDENT>elif notation == IP_OCT:<EOL><INDENT>return _dec_to_oct(dec)<EOL><DEDENT>elif notation == IP_DEC:<EOL><INDENT>return _dec_to_dec_str(dec)<EOL><DEDENT>elif _isnm and notation == NM_BITS:<EOL><INDENT>return _dec_to_bits(dec)<EOL><DEDENT>elif _isnm and notation == NM_WILDCARD:<EOL><INDENT>return _dec_to_wildcard(dec)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % notation_orig)<EOL><DEDENT>

Internally used to convert IPs and netmasks to other notations.


return _is_notation(nm, notation, _isnm=True)<EOL>

Return true if the given netmask is in the given notation.


self.set(self._sub(other), notation=IP_DEC)<EOL>return self<EOL>


def convert_nm(nm, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):

return _convert(nm, notation, inotation, _check=check, _isnm=True)<EOL>


return self.get_ip_number()<EOL>


if check and not is_oct(ip):<EOL><INDENT>raise ValueError('<STR_LIT>' % ip)<EOL><DEDENT>if isinstance(ip, int):<EOL><INDENT>ip = oct(ip)<EOL><DEDENT>return int(str(ip), <NUM_LIT:8>)<EOL>


return self.get()<EOL>


return self._ip_num<EOL>

Return the number of usable IP addresses.


return _NOTATION_KEYS.get(notation, None)<EOL>

Given a numeric value or string value, returns one in IP_DOT, IP_HEX,
    IP_BIN, etc., or None if unable to convert to the internally
    used numeric convention.


bits = []<EOL>while ip:<EOL><INDENT>bits.append(_BYTES_TO_BITS[ip & <NUM_LIT:255>])<EOL>ip >>= <NUM_LIT:8><EOL><DEDENT>bits.reverse()<EOL>return '<STR_LIT>'.join(bits) or <NUM_LIT:32>*'<STR_LIT:0>'<EOL>


return self._nm<EOL>


return '<STR_LIT>'.join(random.SystemRandom().choice(ALPHABET)<EOL>for _ in range(length))<EOL>

Generate a random string of the specified length.

The returned string is composed of an alphabet that shouldn't include any
characters that are easily mistakeable for one another (I, 1, O, 0), and
hopefully won't accidentally contain any English-language curse words.


@classmethod<EOL><INDENT>def coerce(cls, key, value):<DEDENT>

if not isinstance(value, MutableDict):<EOL><INDENT>if isinstance(value, dict):<EOL><INDENT>return MutableDict(value)<EOL><DEDENT>return Mutable.coerce(key, value)<EOL><DEDENT>else:<EOL><INDENT>return value<EOL><DEDENT>

Convert plain dictionaries to MutableDict.


dict.__setitem__(self, key, value)<EOL>self.changed()<EOL>

Detect dictionary set events and emit change events.


@compiles(utcnow)<EOL>def _default_utcnow(element, compiler, **kw):

return "<STR_LIT>"<EOL>

default compilation handler.

    Note that there is no SQL "utcnow()" function; this is a
    "fake" string so that we can produce SQL strings that are dialect-agnostic,
    such as within tests.


<EOL>cols = [(sa.Column(), refcol) for refcol in ref_table.primary_key]<EOL>for col, refcol in cols:<EOL><INDENT>setattr(cls, "<STR_LIT>" % (ref_table.name, refcol.name), col)<EOL><DEDENT>cls.__table__.append_constraint(sa.ForeignKeyConstraint(*zip(*cols)))<EOL>

Create a foreign key reference from the local class to the given remote
        table.

        Adds column references to the declarative class and adds a
        ForeignKeyConstraint.


dbsession = session_factory()<EOL>zope.sqlalchemy.register(<EOL>dbsession, transaction_manager=transaction_manager)<EOL>return dbsession<EOL>

Get a ``sqlalchemy.orm.Session`` instance backed by a transaction.

This function will hook the session to the transaction manager which
will take care of committing any changes.

- When using pyramid_tm it will automatically be committed or aborted
  depending on whether an exception is raised.

- When using scripts you should wrap the session in a manager yourself.
  For example::

      import transaction

      engine = get_engine(settings)
      session_factory = get_session_factory(engine)
      with transaction.manager:
          dbsession = get_tm_session(session_factory, transaction.manager)


@declared_attr<EOL>def m2o(cls):<EOL><INDENT>cls._references((cls.__name__, clsname))<EOL>return relationship(clsname, **kw)<EOL><DEDENT>return m2o<EOL>

Use an event to build a many-to-one relationship on a class.

    This makes use of the :meth:`.References._reference_table` method
    to generate a full foreign key relationship to the remote table.


if pieces["<STR_LIT>"]:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL>rendered += "<STR_LIT>" % (pieces["<STR_LIT>"], pieces["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL><DEDENT>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>"<EOL><DEDENT>return rendered<EOL>

TAG-DISTANCE-gHEX[-dirty].

    Like 'git describe --tags --dirty --always -long'.
    The distance/hash is unconditional.

    Exceptions:
    1: no tags. HEX[-dirty]  (note: no 'g' prefix)


if pieces["<STR_LIT>"]:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL>if pieces["<STR_LIT>"] or pieces["<STR_LIT>"]:<EOL><INDENT>rendered += plus_or_dot(pieces)<EOL>rendered += "<STR_LIT>" % (pieces["<STR_LIT>"], pieces["<STR_LIT>"])<EOL>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>"<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>rendered = "<STR_LIT>" % (pieces["<STR_LIT>"],<EOL>pieces["<STR_LIT>"])<EOL>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>"<EOL><DEDENT><DEDENT>return rendered<EOL>

Build up version string, with post-release "local version identifier".

    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty

    Exceptions:
    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]


def decorate(f):<EOL><INDENT>"""<STR_LIT>"""<EOL>if vcs not in HANDLERS:<EOL><INDENT>HANDLERS[vcs] = {}<EOL><DEDENT>HANDLERS[vcs][method] = f<EOL>return f<EOL><DEDENT>return decorate<EOL>

Decorator to mark a method as the handler for a particular VCS.


@register_vcs_handler("<STR_LIT>", "<STR_LIT>")<EOL>def git_versions_from_keywords(keywords, tag_prefix, verbose):

if not keywords:<EOL><INDENT>raise NotThisMethod("<STR_LIT>")<EOL><DEDENT>date = keywords.get("<STR_LIT:date>")<EOL>if date is not None:<EOL><INDENT>date = date.strip().replace("<STR_LIT:U+0020>", "<STR_LIT:T>", <NUM_LIT:1>).replace("<STR_LIT:U+0020>", "<STR_LIT>", <NUM_LIT:1>)<EOL><DEDENT>refnames = keywords["<STR_LIT>"].strip()<EOL>if refnames.startswith("<STR_LIT>"):<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>raise NotThisMethod("<STR_LIT>")<EOL><DEDENT>refs = set([r.strip() for r in refnames.strip("<STR_LIT>").split("<STR_LIT:U+002C>")])<EOL>TAG = "<STR_LIT>"<EOL>tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])<EOL>if not tags:<EOL><INDENT>tags = set([r for r in refs if re.search(r'<STR_LIT>', r)])<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>" % "<STR_LIT:U+002C>".join(refs - tags))<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>print("<STR_LIT>" % "<STR_LIT:U+002C>".join(sorted(tags)))<EOL><DEDENT>for ref in sorted(tags):<EOL><INDENT>if ref.startswith(tag_prefix):<EOL><INDENT>r = ref[len(tag_prefix):]<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>" % r)<EOL><DEDENT>return {"<STR_LIT:version>": r,<EOL>"<STR_LIT>": keywords["<STR_LIT>"].strip(),<EOL>"<STR_LIT>": False, "<STR_LIT:error>": None,<EOL>"<STR_LIT:date>": date}<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return {"<STR_LIT:version>": "<STR_LIT>",<EOL>"<STR_LIT>": keywords["<STR_LIT>"].strip(),<EOL>"<STR_LIT>": False, "<STR_LIT:error>": "<STR_LIT>", "<STR_LIT:date>": None}<EOL>

Get version information from git keywords.


if pieces["<STR_LIT>"]:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>" % (pieces["<STR_LIT>"], pieces["<STR_LIT>"])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL><DEDENT>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>"<EOL><DEDENT>return rendered<EOL>

TAG[-DISTANCE-gHEX][-dirty].

    Like 'git describe --tags --dirty --always'.

    Exceptions:
    1: no tags. HEX[-dirty]  (note: no 'g' prefix)


<EOL>keywords = {}<EOL>try:<EOL><INDENT>f = open(versionfile_abs, "<STR_LIT:r>")<EOL>for line in f.readlines():<EOL><INDENT>if line.strip().startswith("<STR_LIT>"):<EOL><INDENT>mo = re.search(r'<STR_LIT>', line)<EOL>if mo:<EOL><INDENT>keywords["<STR_LIT>"] = mo.group(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if line.strip().startswith("<STR_LIT>"):<EOL><INDENT>mo = re.search(r'<STR_LIT>', line)<EOL>if mo:<EOL><INDENT>keywords["<STR_LIT>"] = mo.group(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if line.strip().startswith("<STR_LIT>"):<EOL><INDENT>mo = re.search(r'<STR_LIT>', line)<EOL>if mo:<EOL><INDENT>keywords["<STR_LIT:date>"] = mo.group(<NUM_LIT:1>)<EOL><DEDENT><DEDENT><DEDENT>f.close()<EOL><DEDENT>except EnvironmentError:<EOL><INDENT>pass<EOL><DEDENT>return keywords<EOL>

Extract version information from the given file.


<EOL>cfg = VersioneerConfig()<EOL>cfg.VCS = "<STR_LIT>"<EOL>cfg.style = "<STR_LIT>"<EOL>cfg.tag_prefix = "<STR_LIT>"<EOL>cfg.parentdir_prefix = "<STR_LIT>"<EOL>cfg.versionfile_source = "<STR_LIT>"<EOL>cfg.verbose = False<EOL>return cfg<EOL>

Create, populate and return the VersioneerConfig() object.


<EOL>git_refnames = "<STR_LIT>"<EOL>git_full = "<STR_LIT>"<EOL>git_date = "<STR_LIT>"<EOL>keywords = {"<STR_LIT>": git_refnames, "<STR_LIT>": git_full, "<STR_LIT:date>": git_date}<EOL>return keywords<EOL>

Get the keywords needed to look up the version information.


result = re.match(line_pattern, line)<EOL>if result:<EOL><INDENT>_, lineno, offset, severity, msg = result.groups()<EOL>lineno = int(lineno or <NUM_LIT:1>)<EOL>offset = int(offset or <NUM_LIT:0>)<EOL>errno = <NUM_LIT:2><EOL>if severity == '<STR_LIT:error>':<EOL><INDENT>errno = <NUM_LIT:1><EOL><DEDENT>diag = {<EOL>'<STR_LIT:source>': '<STR_LIT>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:start>': {'<STR_LIT>': lineno - <NUM_LIT:1>, '<STR_LIT>': offset},<EOL>'<STR_LIT:end>': {'<STR_LIT>': lineno - <NUM_LIT:1>, '<STR_LIT>': offset + <NUM_LIT:1>}<EOL>},<EOL>'<STR_LIT:message>': msg,<EOL>'<STR_LIT>': errno<EOL>}<EOL>if document:<EOL><INDENT>word = document.word_at_position(diag['<STR_LIT>']['<STR_LIT:start>'])<EOL>if word:<EOL><INDENT>diag['<STR_LIT>']['<STR_LIT:end>']['<STR_LIT>'] = (<EOL>diag['<STR_LIT>']['<STR_LIT:start>']['<STR_LIT>'] + len(word))<EOL><DEDENT><DEDENT>return diag<EOL><DEDENT>

Return a language-server diagnostic from a line of the Mypy error report;
optionally, use the whole document to provide more context on it.


try:<EOL><INDENT>with open(filename) as f:<EOL><INDENT>contents = f.read()<EOL><DEDENT><DEDENT>except EnvironmentError:<EOL><INDENT>raise NotThisMethod("<STR_LIT>")<EOL><DEDENT>mo = re.search(r"<STR_LIT>",<EOL>contents, re.M | re.S)<EOL>if not mo:<EOL><INDENT>mo = re.search(r"<STR_LIT>",<EOL>contents, re.M | re.S)<EOL><DEDENT>if not mo:<EOL><INDENT>raise NotThisMethod("<STR_LIT>")<EOL><DEDENT>return json.loads(mo.group(<NUM_LIT:1>))<EOL>

Try to determine the version from _version.py if present.


if pieces["<STR_LIT>"]:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL>if pieces["<STR_LIT>"] or pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>" % pieces["<STR_LIT>"]<EOL>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>"<EOL><DEDENT>rendered += plus_or_dot(pieces)<EOL>rendered += "<STR_LIT>" % pieces["<STR_LIT>"]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>rendered = "<STR_LIT>" % pieces["<STR_LIT>"]<EOL>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>"<EOL><DEDENT>rendered += "<STR_LIT>" % pieces["<STR_LIT>"]<EOL><DEDENT>return rendered<EOL>

TAG[.postDISTANCE[.dev0]+gHEX] .

    The ".dev0" means dirty. Note that .dev0 sorts backwards
    (a dirty tree will appear "older" than the corresponding clean one),
    but you shouldn't be releasing software with -dirty anyways.

    Exceptions:
    1: no tags. 0.postDISTANCE[.dev0]


found = set()<EOL>setters = False<EOL>errors = <NUM_LIT:0><EOL>with open("<STR_LIT>", "<STR_LIT:r>") as f:<EOL><INDENT>for line in f.readlines():<EOL><INDENT>if "<STR_LIT>" in line:<EOL><INDENT>found.add("<STR_LIT>")<EOL><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>found.add("<STR_LIT>")<EOL><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>found.add("<STR_LIT>")<EOL><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>setters = True<EOL><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>setters = True<EOL><DEDENT><DEDENT><DEDENT>if len(found) != <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>errors += <NUM_LIT:1><EOL><DEDENT>if setters:<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>errors += <NUM_LIT:1><EOL><DEDENT>return errors<EOL>

Validate the contents of setup.py against Versioneer's expectations.


if "<STR_LIT:+>" in pieces.get("<STR_LIT>", "<STR_LIT>"):<EOL><INDENT>return "<STR_LIT:.>"<EOL><DEDENT>return "<STR_LIT:+>"<EOL>

Return a + if we don't already have one, else return a .


def do_vcs_install(manifest_in, versionfile_source, ipy):

GITS = ["<STR_LIT>"]<EOL>if sys.platform == "<STR_LIT:win32>":<EOL><INDENT>GITS = ["<STR_LIT>", "<STR_LIT>"]<EOL><DEDENT>files = [manifest_in, versionfile_source]<EOL>if ipy:<EOL><INDENT>files.append(ipy)<EOL><DEDENT>try:<EOL><INDENT>me = __file__<EOL>if me.endswith("<STR_LIT>") or me.endswith("<STR_LIT>"):<EOL><INDENT>me = os.path.splitext(me)[<NUM_LIT:0>] + "<STR_LIT>"<EOL><DEDENT>versioneer_file = os.path.relpath(me)<EOL><DEDENT>except NameError:<EOL><INDENT>versioneer_file = "<STR_LIT>"<EOL><DEDENT>files.append(versioneer_file)<EOL>present = False<EOL>try:<EOL><INDENT>f = open("<STR_LIT>", "<STR_LIT:r>")<EOL>for line in f.readlines():<EOL><INDENT>if line.strip().startswith(versionfile_source):<EOL><INDENT>if "<STR_LIT>" in line.strip().split()[<NUM_LIT:1>:]:<EOL><INDENT>present = True<EOL><DEDENT><DEDENT><DEDENT>f.close()<EOL><DEDENT>except EnvironmentError:<EOL><INDENT>pass<EOL><DEDENT>if not present:<EOL><INDENT>f = open("<STR_LIT>", "<STR_LIT>")<EOL>f.write("<STR_LIT>" % versionfile_source)<EOL>f.close()<EOL>files.append("<STR_LIT>")<EOL><DEDENT>run_command(GITS, ["<STR_LIT>", "<STR_LIT>"] + files)<EOL>

Git-specific installation logic for Versioneer.

    For Git, this means creating/changing .gitattributes to mark _version.py
    for export-subst keyword substitution.


if pieces["<STR_LIT:error>"]:<EOL><INDENT>return {"<STR_LIT:version>": "<STR_LIT>",<EOL>"<STR_LIT>": pieces.get("<STR_LIT>"),<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT:error>": pieces["<STR_LIT:error>"],<EOL>"<STR_LIT:date>": None}<EOL><DEDENT>if not style or style == "<STR_LIT:default>":<EOL><INDENT>style = "<STR_LIT>"  <EOL><DEDENT>if style == "<STR_LIT>":<EOL><INDENT>rendered = render_pep440(pieces)<EOL><DEDENT>elif style == "<STR_LIT>":<EOL><INDENT>rendered = render_pep440_pre(pieces)<EOL><DEDENT>elif style == "<STR_LIT>":<EOL><INDENT>rendered = render_pep440_post(pieces)<EOL><DEDENT>elif style == "<STR_LIT>":<EOL><INDENT>rendered = render_pep440_old(pieces)<EOL><DEDENT>elif style == "<STR_LIT>":<EOL><INDENT>rendered = render_git_describe(pieces)<EOL><DEDENT>elif style == "<STR_LIT>":<EOL><INDENT>rendered = render_git_describe_long(pieces)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % style)<EOL><DEDENT>return {"<STR_LIT:version>": rendered, "<STR_LIT>": pieces["<STR_LIT>"],<EOL>"<STR_LIT>": pieces["<STR_LIT>"], "<STR_LIT:error>": None,<EOL>"<STR_LIT:date>": pieces.get("<STR_LIT:date>")}<EOL>

Render the given version pieces into the requested style.


os.unlink(filename)<EOL>contents = json.dumps(versions, sort_keys=True,<EOL>indent=<NUM_LIT:1>, separators=("<STR_LIT:U+002C>", "<STR_LIT>"))<EOL>with open(filename, "<STR_LIT:w>") as f:<EOL><INDENT>f.write(SHORT_VERSION_PY % contents)<EOL><DEDENT>print("<STR_LIT>" % (filename, versions["<STR_LIT:version>"]))<EOL>

Write the given version number to the given _version.py file.


if pieces["<STR_LIT>"]:<EOL><INDENT>rendered = pieces["<STR_LIT>"]<EOL>if pieces["<STR_LIT>"]:<EOL><INDENT>rendered += "<STR_LIT>" % pieces["<STR_LIT>"]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>rendered = "<STR_LIT>" % pieces["<STR_LIT>"]<EOL><DEDENT>return rendered<EOL>

TAG[.post.devDISTANCE] -- No -dirty.

    Exceptions:
    1: no tags. 0.post.devDISTANCE


<EOL>setup_cfg = os.path.join(root, "<STR_LIT>")<EOL>parser = configparser.SafeConfigParser()<EOL>with open(setup_cfg, "<STR_LIT:r>") as f:<EOL><INDENT>parser.readfp(f)<EOL><DEDENT>VCS = parser.get("<STR_LIT>", "<STR_LIT>")  <EOL>def get(parser, name):<EOL><INDENT>if parser.has_option("<STR_LIT>", name):<EOL><INDENT>return parser.get("<STR_LIT>", name)<EOL><DEDENT>return None<EOL><DEDENT>cfg = VersioneerConfig()<EOL>cfg.VCS = VCS<EOL>cfg.style = get(parser, "<STR_LIT>") or "<STR_LIT>"<EOL>cfg.versionfile_source = get(parser, "<STR_LIT>")<EOL>cfg.versionfile_build = get(parser, "<STR_LIT>")<EOL>cfg.tag_prefix = get(parser, "<STR_LIT>")<EOL>if cfg.tag_prefix in ("<STR_LIT>", '<STR_LIT>'):<EOL><INDENT>cfg.tag_prefix = "<STR_LIT>"<EOL><DEDENT>cfg.parentdir_prefix = get(parser, "<STR_LIT>")<EOL>cfg.verbose = get(parser, "<STR_LIT>")<EOL>return cfg<EOL>

Read the project setup.cfg file to determine Versioneer config.


if "<STR_LIT>" in sys.modules:<EOL><INDENT>del sys.modules["<STR_LIT>"]<EOL><DEDENT>root = get_root()<EOL>cfg = get_config_from_root(root)<EOL>assert cfg.VCS is not None, "<STR_LIT>"<EOL>handlers = HANDLERS.get(cfg.VCS)<EOL>assert handlers, "<STR_LIT>" % cfg.VCS<EOL>verbose = verbose or cfg.verbose<EOL>assert cfg.versionfile_source is not None,"<STR_LIT>"<EOL>assert cfg.tag_prefix is not None, "<STR_LIT>"<EOL>versionfile_abs = os.path.join(root, cfg.versionfile_source)<EOL>get_keywords_f = handlers.get("<STR_LIT>")<EOL>from_keywords_f = handlers.get("<STR_LIT>")<EOL>if get_keywords_f and from_keywords_f:<EOL><INDENT>try:<EOL><INDENT>keywords = get_keywords_f(versionfile_abs)<EOL>ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>" % ver)<EOL><DEDENT>return ver<EOL><DEDENT>except NotThisMethod:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>ver = versions_from_file(versionfile_abs)<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>" % (versionfile_abs, ver))<EOL><DEDENT>return ver<EOL><DEDENT>except NotThisMethod:<EOL><INDENT>pass<EOL><DEDENT>from_vcs_f = handlers.get("<STR_LIT>")<EOL>if from_vcs_f:<EOL><INDENT>try:<EOL><INDENT>pieces = from_vcs_f(cfg.tag_prefix, root, verbose)<EOL>ver = render(pieces, cfg.style)<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>" % ver)<EOL><DEDENT>return ver<EOL><DEDENT>except NotThisMethod:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>if cfg.parentdir_prefix:<EOL><INDENT>ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>" % ver)<EOL><DEDENT>return ver<EOL><DEDENT><DEDENT>except NotThisMethod:<EOL><INDENT>pass<EOL><DEDENT>if verbose:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return {"<STR_LIT:version>": "<STR_LIT>", "<STR_LIT>": None,<EOL>"<STR_LIT>": None, "<STR_LIT:error>": "<STR_LIT>",<EOL>"<STR_LIT:date>": None}<EOL>

Get the project version from whatever source is available.

    Returns dict with two keys: 'version' and 'full'.


if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):<EOL><INDENT>return obj.strftime(default_date_fmt)<EOL><DEDENT>else:<EOL><INDENT>return str(obj)<EOL><DEDENT>

Coerce everything to strings.
    All objects representing time get output according to default_date_fmt.


pass<EOL>


def init_logs(self, path=None, target=None, level=None,<EOL>server_hostname=None, extra=None):

path = path if path is not None else self.path<EOL>target = target if target is not None else self.target<EOL>level = level if level is not None else logging.DEBUG<EOL>server_hostname = (server_hostname if server_hostname is not None<EOL>else '<STR_LIT:localhost>')<EOL>extra = extra if extra is not None else {'<STR_LIT>': None}<EOL>init_logs(path=path, target=target, level=level,<EOL>server_hostname=server_hostname, fields=extra)<EOL>

Simple canned way to initialize pyzlog.

        Initialize pyslog for tests. If path or target are not
        specified, will default to path and target properties on the
        object. leve will default to logging.DEBUG, server_hostname
        defaults to localhost, and extra defaults to {'extra': None}

        :param path: path to find the log file
        :param target: name of the log file
        :param level: log level for this instance
        :param server_hostname: hostname to put in each entry
        :param extra: whitelist/defaults of extra fields to add to each entry
        :type path: string
        :type target: string
        :type level: int
        :type server_hostname: string
        :type extra: dict


log_file = os.path.abspath(<EOL>os.path.join(path, target))<EOL>logger = logging.getLogger(logger_name)<EOL>logger.setLevel(level)<EOL>handler = logging.handlers.RotatingFileHandler(<EOL>log_file, maxBytes=maxBytes, backupCount=backupCount)<EOL>handler.setLevel(level)<EOL>handler.setFormatter(<EOL>JsonFormatter(<EOL>application_name=application_name,<EOL>server_hostname=server_hostname,<EOL>fields=fields))<EOL>logger.addHandler(handler)<EOL>

Initialize the zlogger.

    Sets up a rotating file handler to the specified path and file with
    the given size and backup count limits, sets the default
    application_name, server_hostname, and default/whitelist fields.

    :param path: path to write the log file
    :param target: name of the log file
    :param logger_name: name of the logger (defaults to root)
    :param level: log level for this logger (defaults to logging.DEBUG)
    :param maxBytes: size of the file before rotation (default 1MB)
    :param application_name: app name to add to each log entry
    :param server_hostname: hostname to add to each log entry
    :param fields: default/whitelist fields.
    :type path: string
    :type target: string
    :type logger_name: string
    :type level: int
    :type maxBytes: int
    :type backupCount: int
    :type application_name: string
    :type server_hostname: string
    :type fields: dict


def render_to_response(self, context, **response_kwargs):

if self.request.is_ajax():<EOL><INDENT>template = self.page_template<EOL><DEDENT>else:<EOL><INDENT>template = self.get_template_names()<EOL><DEDENT>return self.response_class(<EOL>request=self.request,<EOL>template=template,<EOL>context=context,<EOL>**response_kwargs<EOL>)<EOL>

Returns a response with a template depending if the request is ajax
or not and it renders with the given context.


def aggregate_history(self, ip, days_limit=None):

first_date = None<EOL>last_date = None<EOL>prec_asn = None<EOL>prec_block = None<EOL>for entry in self.history(ip, days_limit):<EOL><INDENT>if entry is None:<EOL><INDENT>continue<EOL><DEDENT>date, asn, block = entry<EOL>if first_date is None:<EOL><INDENT>last_date = date<EOL>first_date = date<EOL>prec_asn = asn<EOL>prec_block = block<EOL><DEDENT>elif prec_asn == asn and prec_block == block:<EOL><INDENT>first_date = date<EOL><DEDENT>else:<EOL><INDENT>yield first_date, last_date, prec_asn, prec_block<EOL>last_date = date<EOL>first_date = date<EOL>prec_asn = asn<EOL>prec_block = block<EOL><DEDENT><DEDENT>if first_date is not None:<EOL><INDENT>yield first_date, last_date, prec_asn, prec_block<EOL><DEDENT>

Get the full history of an IP, aggregate the result instead of
returning one line per day.

:param ip: IP address to search for
:param days_limit: Max amount of days to query. (None means no limit)

:rtype: list. For each change: FirstDay, LastDay, ASN, Block


urlretrieve(url, os.path.join(c.raw_data, path_temp_bviewfile))<EOL>os.rename(os.path.join(c.raw_data, path_temp_bviewfile), c.path_bviewfile)<EOL>

Inconditianilly download the URL in a temporary directory.
When finished, the file is moved in the real directory.
Like this an other process will not attempt to extract an inclomplete file.


path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, '<STR_LIT>', filename)<EOL>path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename)<EOL>try:<EOL><INDENT>f = urlopen(url)<EOL><DEDENT>except:<EOL><INDENT>return False<EOL><DEDENT>if f.getcode() != <NUM_LIT:200>:<EOL><INDENT>publisher.warning('<STR_LIT>'.format(url, f.getcode()))<EOL>return False<EOL><DEDENT>try:<EOL><INDENT>with open(path_temp_bviewfile, '<STR_LIT:w>') as outfile:<EOL><INDENT>outfile.write(f.read())<EOL><DEDENT>os.rename(path_temp_bviewfile, path_bviewfile)<EOL><DEDENT>except:<EOL><INDENT>os.remove(path_temp_bviewfile)<EOL>return False<EOL><DEDENT>return True<EOL>


cur_file = os.path.join(c.bview_dir, filename)<EOL>old_file = os.path.join(c.bview_dir, '<STR_LIT>', filename)<EOL>if not os.path.exists(cur_file) and not os.path.exists(old_file):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Verify that the file has not already been downloaded.


pids = []<EOL>while len(filenames) > <NUM_LIT:0>:<EOL><INDENT>while len(filenames) > <NUM_LIT:0> and len(pids) < max_simultaneous_processes:<EOL><INDENT>filename = filenames.pop()<EOL>pids.append(service_start(service=process_name,<EOL>param=['<STR_LIT>', filename, '<STR_LIT>',<EOL>imported_day]))<EOL><DEDENT>while len(pids) == max_simultaneous_processes:<EOL><INDENT>time.sleep(sleep_timer)<EOL>pids = update_running_pids(pids)<EOL><DEDENT><DEDENT>while len(pids) > <NUM_LIT:0>:<EOL><INDENT>time.sleep(sleep_timer)<EOL>pids = update_running_pids(pids)<EOL><DEDENT>

Run processes which push the routing dump of the RIPE in a redis
database.
The dump has been splitted in multiple files and each process run
on one of this files.


ip = request.remote_addr<EOL>ua = request.headers.get('<STR_LIT>', '<STR_LIT>')<EOL>method = request.json.get('<STR_LIT>')<EOL>if method is None:<EOL><INDENT>__query_logging(ip, ua, method, level='<STR_LIT>')<EOL>return json.dumps({'<STR_LIT:error>': '<STR_LIT>'})<EOL><DEDENT>if method not in authorized_methods:<EOL><INDENT>__query_logging(ip, ua, method, level='<STR_LIT>')<EOL>return json.dumps({'<STR_LIT:error>': '<STR_LIT>'})<EOL><DEDENT>fct = globals().get(method)<EOL>if fct is None:<EOL><INDENT>__query_logging(ip, ua, method, level='<STR_LIT>')<EOL>return json.dumps({'<STR_LIT:error>': '<STR_LIT>'})<EOL><DEDENT>if request.json.get('<STR_LIT>') is None:<EOL><INDENT>__query_logging(ip, ua, method, level='<STR_LIT>')<EOL>return json.dumps({'<STR_LIT:error>': '<STR_LIT>'})<EOL><DEDENT>try:<EOL><INDENT>result = fct(request.json)<EOL>__query_logging(ip, ua, method, request.json.get('<STR_LIT>'),<EOL>request.json.get('<STR_LIT>'), request.json.get('<STR_LIT>'))<EOL>return result<EOL><DEDENT>except Exception:<EOL><INDENT>__query_logging(ip, ua, method, request.json.get('<STR_LIT>'), level='<STR_LIT:error>')<EOL>return json.dumps({'<STR_LIT:error>': '<STR_LIT>'})<EOL><DEDENT>

Function called when an query is made on /json. Expects a JSON
object with at least a 'method' entry.


if names is None:<EOL><INDENT>return CompiledFormat(fmt)<EOL><DEDENT>else:<EOL><INDENT>return CompiledFormatDict(fmt, names)<EOL><DEDENT>

Compile given format string `fmt` and return a compiled format
    object that can be used to pack and/or unpack data multiple times.

    Returns a :class:`~bitstruct.CompiledFormat` object if `names` is
    ``None``, and otherwise a :class:`~bitstruct.CompiledFormatDict`
    object.

    See :func:`~bitstruct.pack_dict()` for details on `names`.


def pack_into(fmt, buf, offset, *args, **kwargs):

return CompiledFormat(fmt).pack_into(buf,<EOL>offset,<EOL>*args,<EOL>**kwargs)<EOL>

Pack given values v1, v2, ... into given bytearray `buf`, starting
    at given bit offset `offset`. Pack according to given format
    string `fmt`. Give `fill_padding` as ``False`` to leave padding
    bits in `buf` unmodified.


return CompiledFormat(fmt).pack(*args)<EOL>

Return a bytes object containing the values v1, v2, ... packed
    according to given format string `fmt`. If the total number of
    bits are not a multiple of 8, padding will be added at the end of
    the last byte.

    `fmt` is a string of bitorder-type-length groups, and optionally a
    byteorder identifier after the groups. Bitorder and byteorder may
    be omitted.

    Bitorder is either ``>`` or ``<``, where ``>`` means MSB first and
    ``<`` means LSB first. If bitorder is omitted, the previous
    values' bitorder is used for the current value. For example, in
    the format string ``'u1<u2u3'``, ``u1`` is MSB first and both
    ``u2`` and ``u3`` are LSB first.

    Byteorder is either ``>`` or ``<``, where ``>`` means most
    significant byte first and ``<`` means least significant byte
    first. If byteorder is omitted, most significant byte first is
    used.

    There are eight types; ``u``, ``s``, ``f``, ``b``, ``t``, ``r``,
    ``p`` and ``P``.

    - ``u`` -- unsigned integer
    - ``s`` -- signed integer
    - ``f`` -- floating point number of 16, 32, or 64 bits
    - ``b`` -- boolean
    - ``t`` -- text (ascii or utf-8)
    - ``r`` -- raw, bytes
    - ``p`` -- padding with zeros, ignore
    - ``P`` -- padding with ones, ignore

    Length is the number of bits to pack the value into.

    Example format string with default bit and byte ordering:
    ``'u1u3p7s16'``

    Same format string, but with least significant byte first:
    ``'u1u3p7s16<'``

    Same format string, but with LSB first (``<`` prefix) and least
    significant byte first (``<`` suffix): ``'<u1u3p7s16<'``

    It is allowed to separate groups with a single space for better
    readability.


def pack_into(self, buf, offset, *args, **kwargs):

<EOL>if len(args) < self._number_of_arguments:<EOL><INDENT>raise Error(<EOL>"<STR_LIT>".format(<EOL>self._number_of_arguments,<EOL>len(args)))<EOL><DEDENT>self.pack_into_any(buf, offset, args, **kwargs)<EOL>


def unpack_from(fmt, data, offset=<NUM_LIT:0>):

return CompiledFormat(fmt).unpack_from(data, offset)<EOL>

Unpack `data` (bytes or bytearray) according to given format string
    `fmt`, starting at given bit offset `offset`. The result is a
    tuple even if it contains exactly one item.


def unpack_from_dict(fmt, names, data, offset=<NUM_LIT:0>):

return CompiledFormatDict(fmt, names).unpack_from(data, offset)<EOL>

Same as :func:`~bitstruct.unpack_from_dict()`, but returns a
    dictionary.

    See :func:`~bitstruct.pack_dict()` for details on `names`.


if sys.version_info < (<NUM_LIT:3>, <NUM_LIT:0>) and isinstance(fileobj, file):<EOL><INDENT>filename = fileobj.name<EOL>fileobj = io.FileIO(fileobj.fileno(), closefd=False)<EOL>fileobj.name = filename<EOL><DEDENT>assert isinstance(fileobj, io.IOBase),"<STR_LIT>"% type(fileobj)<EOL>return fileobj if fileobj.seekable()else ArchiveTemp(fileobj)<EOL>

If the file-object is not seekable, return  ArchiveTemp of the fileobject,
otherwise return the file-object itself


fieldstrs = []<EOL>if len(byfields) == <NUM_LIT:1> and isinstance(byfields[<NUM_LIT:0>], type) andissubclass(byfields[<NUM_LIT:0>], SortDirection):<EOL><INDENT>byfields = [byfields[<NUM_LIT:0>](field)]<EOL><DEDENT>for f in byfields:<EOL><INDENT>fieldstrs += [f.field, f.DIRSTRING]<EOL><DEDENT>args = [field]<EOL>if fieldstrs:<EOL><INDENT>args += ['<STR_LIT>'] + fieldstrs<EOL><DEDENT>super(first_value, self).__init__(*args)<EOL>self._field = field<EOL>

Selects the first value of the given field within the group.

### Parameter

- **field**: Source field used for the value
- **byfields**: How to sort the results. This can be either the
    *class* of `aggregation.Asc` or `aggregation.Desc` in which
    case the field `field` is also used as the sort input.

    `byfields` can also be one or more *instances* of `Asc` or `Desc`
    indicating the sort order for these fields


return self.redis.execute_command(self.DROP_CMD, self.index_name)<EOL>


args, query = self._mk_query_args(query)<EOL>st = time.time()<EOL>res = self.redis.execute_command(self.SEARCH_CMD, *args)<EOL>return Result(res,<EOL>not query._no_content,<EOL>duration=(time.time() - st) * <NUM_LIT>,<EOL>has_payload=query._with_payloads)<EOL>

Search the index for a given query, and return a result of documents

### Parameters

- **query**: the search query. Either a text for simple queries with default parameters, or a Query object for complex queries.
             See RediSearch's documentation on query format
- **snippet_sizes**: A dictionary of {field: snippet_size} used to trim and format the result. e.g.e {'body': 500}


def add_document(self, doc_id, nosave=False, score=<NUM_LIT:1.0>, payload=None,<EOL>replace=False, partial=False, language=None, **fields):

return self._add_document(doc_id, conn=None, nosave=nosave, score=score, <EOL>payload=payload, replace=replace,<EOL>partial=partial, language=language, **fields)<EOL>

Add a single document to the index.

### Parameters

- **doc_id**: the id of the saved document.
- **nosave**: if set to true, we just index the document, and don't save a copy of it. This means that searches will just return ids.
- **score**: the document ranking, between 0.0 and 1.0
- **payload**: optional inner-index payload we can save for fast access in scoring functions
- **replace**: if True, and the document already is in the index, we perform an update and reindex the document
- **partial**: if True, the fields specified will be added to the existing document.
               This has the added benefit that any fields specified with `no_index`
               will not be reindexed again. Implies `replace`
- **language**: Specify the language used for document tokenization.
- **fields** kwargs dictionary of the document fields to be saved and/or indexed.
             NOTE: Geo points shoule be encoded as strings of "lon,lat"


self._in_order = True<EOL>return self<EOL>

Match only documents where the query terms appear in the same order in the document.
i.e. for the query 'hello world', we do not match 'world hello'


self._slop = slop<EOL>return self<EOL>

Allow a masimum of N intervening non matched terms between phrase terms (0 means exact phrase)


self._offset = offset<EOL>self._num = num<EOL>return self<EOL>

Set the paging for the query (defaults to 0..10).

- **offset**: Paging offset for the results. Defaults to 0
- **num**: How many results do we want


args = [self._query_string]<EOL>if self._no_content:<EOL><INDENT>args.append('<STR_LIT>')<EOL><DEDENT>if self._fields:<EOL><INDENT>args.append('<STR_LIT>')<EOL>args.append(len(self._fields))<EOL>args += self._fields<EOL><DEDENT>if self._verbatim:<EOL><INDENT>args.append('<STR_LIT>')<EOL><DEDENT>if self._no_stopwords:<EOL><INDENT>args.append('<STR_LIT>')<EOL><DEDENT>if self._filters:<EOL><INDENT>for flt in self._filters:<EOL><INDENT>assert isinstance(flt, Filter)<EOL>args += flt.args<EOL><DEDENT><DEDENT>if self._with_payloads:<EOL><INDENT>args.append('<STR_LIT>')<EOL><DEDENT>if self._ids:<EOL><INDENT>args.append('<STR_LIT>')<EOL>args.append(len(self._ids))<EOL>args += self._ids<EOL><DEDENT>if self._slop >= <NUM_LIT:0>:<EOL><INDENT>args += ['<STR_LIT>', self._slop]<EOL><DEDENT>if self._in_order:<EOL><INDENT>args.append('<STR_LIT>')<EOL><DEDENT>if self._return_fields:<EOL><INDENT>args.append('<STR_LIT>')<EOL>args.append(len(self._return_fields))<EOL>args += self._return_fields<EOL><DEDENT>if self._sortby:<EOL><INDENT>assert isinstance(self._sortby, SortbyField)<EOL>args.append('<STR_LIT>')<EOL>args += self._sortby.args<EOL><DEDENT>if self._language:<EOL><INDENT>args += ['<STR_LIT>', self._language]<EOL><DEDENT>args += self._summarize_fields + self._highlight_fields<EOL>args += ["<STR_LIT>", self._offset, self._num]<EOL>return args<EOL>

Format the redis arguments for this query and return them


def summarize(self, fields=None, context_len=None, num_frags=None, sep=None):

args = ['<STR_LIT>']<EOL>fields = self._mk_field_list(fields)<EOL>if fields:<EOL><INDENT>args += ['<STR_LIT>', str(len(fields))] + fields<EOL><DEDENT>if context_len is not None:<EOL><INDENT>args += ['<STR_LIT>', str(context_len)]<EOL><DEDENT>if num_frags is not None:<EOL><INDENT>args += ['<STR_LIT>', str(num_frags)]<EOL><DEDENT>if sep is not None:<EOL><INDENT>args += ['<STR_LIT>', sep]<EOL><DEDENT>self._summarize_fields = args<EOL>return self<EOL>

Return an abridged format of the field, containing only the segments of
the field which contain the matching term(s).

If `fields` is specified, then only the mentioned fields are
summarized; otherwise all results are summarized.

Server side defaults are used for each option (except `fields`) if not specified

- **fields** List of fields to summarize. All fields are summarized if not specified
- **context_len** Amount of context to include with each fragment
- **num_frags** Number of fragments per document
- **sep** Separator string to separate fragments


self._filters.append(flt)<EOL>return self<EOL>

Add a numeric or geo filter to the query.
**Currently only one of each filter is supported by the engine**

- **flt**: A NumericFilter or GeoFilter object, used on a corresponding field


def highlight(self, fields=None, tags=None):

args = ['<STR_LIT>']<EOL>fields = self._mk_field_list(fields)<EOL>if fields:<EOL><INDENT>args += ['<STR_LIT>', str(len(fields))] + fields<EOL><DEDENT>if tags:<EOL><INDENT>args += ['<STR_LIT>'] + list(tags)<EOL><DEDENT>self._highlight_fields = args<EOL>return self<EOL>

Apply specified markup to matched term(s) within the returned field(s)

- **fields** If specified then only those mentioned fields are highlighted, otherwise all fields are highlighted
- **tags** A list of two strings to surround the match.


self._query_string = query_string<EOL>self._offset = <NUM_LIT:0><EOL>self._num = <NUM_LIT:10><EOL>self._no_content = False<EOL>self._no_stopwords = False<EOL>self._fields = None<EOL>self._verbatim = False<EOL>self._with_payloads = False<EOL>self._filters = list()<EOL>self._ids = None<EOL>self._slop = -<NUM_LIT:1><EOL>self._in_order = False<EOL>self._sortby = None<EOL>self._return_fields = []<EOL>self._summarize_fields = []<EOL>self._highlight_fields = []<EOL>self._language = None<EOL>

Create a new query object.
The query string is set in the constructor, and other options have setter functions.


return self._query_string<EOL>

Return the query string of this query only


self._sortby = SortbyField(field, asc)<EOL>return self<EOL>

Add a sortby field to the query

- **field** - the name of the field to sort by
- **asc** - when `True`, sorting will be done in asceding order


self._ids = ids<EOL>return self<EOL>

Limit the results to a specific set of pre-known document ids of any length


return between(None, n, inclusive_max=False)<EOL>


return self.redis.execute_command(AutoCompleter.SUGDEL_COMMAND, self.key, string)<EOL>

Delete a string from the AutoCompleter index.
Returns 1 if the string was found and deleted, 0 otherwise


def __init__(self, key, host='<STR_LIT:localhost>', port=<NUM_LIT>, conn = None):

self.key = key<EOL>self.redis = conn if conn is not None else Redis(<EOL>connection_pool = ConnectionPool(host=host, port=port))<EOL>

Create a new AutoCompleter client for the given key, and optional host and port

If conn is not None, we employ an already existing redis connection


self._loadfields.extend(fields)<EOL>return self<EOL>

Indicate the fields to be returned in the response. These fields are
returned in addition to any others implicitly specified.

### Parameters

- **fields**: One or more fields in the format of `@field`


self._max = kwargs.get('<STR_LIT>', <NUM_LIT:0>)<EOL>if isinstance(fields, (string_types, SortDirection)):<EOL><INDENT>fields = [fields]<EOL><DEDENT>for f in fields:<EOL><INDENT>if isinstance(f, SortDirection):<EOL><INDENT>self._sortby += [f.field, f.DIRSTRING]<EOL><DEDENT>else:<EOL><INDENT>self._sortby.append(f)<EOL><DEDENT><DEDENT>return self<EOL>

Indicate how the results should be sorted. This can also be used for
*top-N* style queries

### Parameters

- **fields**: The fields by which to sort. This can be either a single
    field or a list of fields. If you wish to specify order, you can
    use the `Asc` or `Desc` wrapper classes.
- **max**: Maximum number of results to return. This can be used instead
    of `LIMIT` and is also faster.

Example of sorting by `foo` ascending and `bar` descending:

```
sort_by(Asc('@foo'), Desc('@bar'))
```

Return the top 10 customers:

```
AggregateRequest()\
    .group_by('@customer', r.sum('@paid').alias(FIELDNAME))\
    .sort_by(Desc('@paid'), max=10)
```


attrs = get_attributes(var)<EOL>pprint(attrs)<EOL>return attrs<EOL>

Given a variable in the template's context, print and return the list of
attributes thare accessible inside of the template. For example, private
attributes or callables that require arguments are excluded.


try:<EOL><INDENT>import ipdb as pdb<EOL><DEDENT>except ImportError:<EOL><INDENT>import pdb<EOL>print("<STR_LIT>")<EOL><DEDENT>print("<STR_LIT>")<EOL>render = lambda s: template.Template(s).render(context)<EOL>availables = get_variables(context)<EOL>pprint(availables)<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>for var in availables:<EOL><INDENT>locals()[var] = context[var]<EOL><DEDENT>pdb.set_trace()<EOL>return '<STR_LIT>'<EOL>

Start a pdb set_trace inside of the template with the context available as
'context'. Uses ipdb if available.


meta_keys = (key for key in list(var_data.keys())<EOL>if key.startswith('<STR_LIT>'))<EOL>for key in meta_keys:<EOL><INDENT>display_key = key[<NUM_LIT:5>:].capitalize()<EOL>pprint('<STR_LIT>'.format(display_key, var_data.pop(key)))<EOL><DEDENT>pprint(var_data)<EOL>

Given a dictionary of variable attribute data from get_details display the
data in the terminal.


is_valid = partial(is_valid_in_template, var)<EOL>return list(filter(is_valid, dir(var)))<EOL>

Given a varaible, return the list of attributes that are available inside
of a template


value = getattr(var, attr)<EOL>kls = getattr(getattr(value, '<STR_LIT>', '<STR_LIT>'), '<STR_LIT>', '<STR_LIT>')<EOL>if kls in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>return kls<EOL><DEDENT>if callable(value):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return value<EOL>

Given a variable and one of its attributes that are available inside of
a template, return its 'method' if it is a callable, its class name if it
is a model manager, otherwise return its value


var_data = {}<EOL>module = getattr(var, '<STR_LIT>', '<STR_LIT>')<EOL>kls = getattr(getattr(var, '<STR_LIT>', '<STR_LIT>'), '<STR_LIT>', '<STR_LIT>')<EOL>if module:<EOL><INDENT>var_data['<STR_LIT>'] = module<EOL><DEDENT>if kls:<EOL><INDENT>var_data['<STR_LIT>'] = kls<EOL><DEDENT>for attr in get_attributes(var):<EOL><INDENT>value = _get_detail_value(var, attr)<EOL>if value is not None:<EOL><INDENT>var_data[attr] = value<EOL><DEDENT><DEDENT>return var_data<EOL>

Given a variable inside the context, obtain the attributes/callables,
their values where possible, and the module name and class name if possible


for i in iterable:<EOL><INDENT>if isinstance(i, Iterable) and not isinstance(i, string_types):<EOL><INDENT>for sub_i in _flatten(i):<EOL><INDENT>yield sub_i<EOL><DEDENT><DEDENT>else:<EOL><INDENT>yield i<EOL><DEDENT><DEDENT>

Given an iterable with nested iterables, generate a flat iterable


def volcano(differential_dfs, title='<STR_LIT>', scripts_mode="<STR_LIT>", data_mode="<STR_LIT>",<EOL>organism="<STR_LIT>", q_value_column_name="<STR_LIT:q>", log2FC_column_name="<STR_LIT>",<EOL>output_dir="<STR_LIT:.>", filename="<STR_LIT>", version=this_version):

output_dir = Path(output_dir)<EOL>output_dir.mkdir(exist_ok=True, parents=True)<EOL>if isinstance(differential_dfs, pd.DataFrame):<EOL><INDENT>differential_dfs = {'<STR_LIT>': differential_dfs}<EOL><DEDENT>for name, df in differential_dfs.items():<EOL><INDENT>df = df[[q_value_column_name, log2FC_column_name]]<EOL>df.columns = ['<STR_LIT:q>', '<STR_LIT>']<EOL>df = df.round(<NUM_LIT:2>)<EOL>_verify_differential_df(df)<EOL>del differential_dfs[name]<EOL>differential_dfs[_sanitize(name)] = df<EOL><DEDENT>names_and_differentials = f"<STR_LIT>"<EOL>data_block = _data_block(data_mode, [('<STR_LIT>', names_and_differentials)], output_dir, include_gene_sets=False, organism=organism)<EOL>scripts = third_party_scripts + [CDN_url(version)+"<STR_LIT>", CDN_url(version)+"<STR_LIT>", CDN_url(version)+"<STR_LIT>"]<EOL>scripts_block = _scripts_block(scripts, scripts_mode, output_dir)<EOL>html = templateEnv.get_template('<STR_LIT>').render(title=title, scripts_block=scripts_block+'<STR_LIT:\n>'+data_block, organism="<STR_LIT>")<EOL>(output_dir / filename).write_text(html)<EOL>return (output_dir / filename).resolve()<EOL>

Arguments:
    differential_dfs (dict or pandas.DataFrame): python dict of names to pandas dataframes, or a single dataframe, indexed by gene symbols which must have columns named log2FC and qval.
    title (str): The title of the plot (to be embedded in the html).
    scripts_mode (str): Choose from [`"CDN"`, `"directory"`, `"inline"`]:

        - `"CDN"` compiles a single HTML page with links to scripts hosted on a CDN,

        - `"directory"` compiles a directory with all scripts locally cached,

        - `"inline"` compiles a single HTML file with all scripts/styles inlined.

    data_mode (str): Choose from ["directory", "inline"]:

        - "directory" compiles a directory with all data locally cached,

        - "inline" compiles a single HTML file with all data inlined.

    organism (str): `"human"` or `"mouse"`
    q_value_column_name (str):
    log2FC_column_name (str):
    output_dir (str): the directory in which to output the file
    filename (str): the filename of the output file
    version (str): the version of the javascripts to use.
        Leave the default to pin the version, or choose "latest" to get updates,
        or choose part of the version string to get minor updates.
Returns:
    Path: The filepath which the html was outputted to.


def heatmap(genes_by_samples_matrix, sample_attributes, title='<STR_LIT>', scripts_mode="<STR_LIT>", data_mode="<STR_LIT>",<EOL>organism="<STR_LIT>", separate_zscore_by=["<STR_LIT>"],<EOL>output_dir="<STR_LIT:.>", filename="<STR_LIT>", version=this_version):

output_dir = Path(output_dir)<EOL>output_dir.mkdir(exist_ok=True, parents=True)<EOL>_verify_sample_by_genes_matrix(genes_by_samples_matrix)<EOL>_verify_sample_attributes(genes_by_samples_matrix, sample_attributes)<EOL>genes_by_samples_matrix = genes_by_samples_matrix.round(<NUM_LIT:2>)<EOL>matrix = f"<STR_LIT>"<EOL>classes = f"<STR_LIT>"<EOL>data_block = _data_block(data_mode, [('<STR_LIT>', matrix), ('<STR_LIT>', classes)], output_dir, organism=organism)<EOL>scripts = third_party_scripts + [CDN_url(version)+"<STR_LIT>", CDN_url(version)+"<STR_LIT>", CDN_url(version)+"<STR_LIT>"]<EOL>scripts_block = _scripts_block(scripts, scripts_mode, output_dir)<EOL>html = templateEnv.get_template('<STR_LIT>').render(title=title, scripts_block=scripts_block+'<STR_LIT:\n>'+data_block, separate_zscore_by=separate_zscore_by)<EOL>(output_dir / filename).write_text(html)<EOL>return (output_dir / filename).resolve()<EOL>

Arguments:
    genes_by_samples_matrix (pandas.DataFrame): dataframe indexed by genes, columns are samples
    sample_attributes (pandas.DataFrame): dataframe indexed by samples, columns are sample attributes (e.g. classes)
    title (str): The title of the plot (to be embedded in the html).
    scripts_mode (str): Choose from [`"CDN"`, `"directory"`, `"inline"`]:

        - `"CDN"` compiles a single HTML page with links to scripts hosted on a CDN,

        - `"directory"` compiles a directory with all scripts locally cached,

        - `"inline"` compiles a single HTML file with all scripts/styles inlined.

    data_mode (str): Choose from ["directory", "inline"]:

        - "directory" compiles a directory with all data locally cached,

        - "inline" compiles a single HTML file with all data inlined.

    organism (str): `"human"` or `"mouse"`
    separate_zscore_by (list):
    output_dir (str): the directory in which to output the file
    filename (str): the filename of the output file
    version (str): the version of the javascripts to use.
        Leave the default to pin the version, or choose "latest" to get updates,
        or choose part of the version string to get minor updates.
Returns:
    Path: The filepath which the html was outputted to.


def graph(networkx_graph, title='<STR_LIT>', scripts_mode="<STR_LIT>", data_mode="<STR_LIT>",<EOL>output_dir="<STR_LIT:.>", filename="<STR_LIT>", version=this_version):

output_dir = Path(output_dir)<EOL>output_dir.mkdir(exist_ok=True, parents=True)<EOL>scripts = third_party_scripts + [CDN_url(version)+"<STR_LIT>", CDN_url(version)+"<STR_LIT>"]<EOL>scripts_block = _scripts_block(scripts, scripts_mode, output_dir)<EOL>graph_json = nx_json.node_link_data(networkx_graph)<EOL>for node in graph_json['<STR_LIT>']:<EOL><INDENT>for attr, val in node.items():<EOL><INDENT>if isinstance(val, numbers.Number):<EOL><INDENT>node[attr] = round(val, <NUM_LIT:2>)<EOL><DEDENT><DEDENT><DEDENT>for link in graph_json['<STR_LIT>']:<EOL><INDENT>for attr, val in link.items():<EOL><INDENT>if isinstance(val, numbers.Number):<EOL><INDENT>link[attr] = round(val, <NUM_LIT:2>)<EOL><DEDENT><DEDENT><DEDENT>graph_json = f"<STR_LIT>"<EOL>data_block = _data_block(data_mode, [('<STR_LIT>', graph_json)], output_dir)<EOL>html = templateEnv.get_template('<STR_LIT>').render(title=title, scripts_block=scripts_block+'<STR_LIT:\n>'+data_block, nodes=networkx_graph.nodes())<EOL>(output_dir / filename).write_text(html)<EOL>return (output_dir / filename).resolve()<EOL>

Arguments:
    networkx_graph (networkx.Graph): any instance of networkx.Graph
    title (str): The title of the plot (to be embedded in the html).
    scripts_mode (str): Choose from [`"CDN"`, `"directory"`, `"inline"`]:

        - `"CDN"` compiles a single HTML page with links to scripts hosted on a CDN,

        - `"directory"` compiles a directory with all scripts locally cached,

        - `"inline"` compiles a single HTML file with all scripts/styles inlined.

    data_mode (str): Choose from ["directory", "inline"]:

        - "directory" compiles a directory with all data locally cached,

        - "inline" compiles a single HTML file with all data inlined.

    output_dir (str): the directory in which to output the file
    filename (str): the filename of the output file
    version (str): the version of the javascripts to use.
        Leave the default to pin the version, or choose "latest" to get updates,
        or choose part of the version string to get minor updates.
Returns:
    Path: The filepath which the html was outputted to.


ver = Version()<EOL>ver.conn = self.conn<EOL>ver.attrs = {<EOL>'<STR_LIT>': self.attrs['<STR_LIT:id>'],<EOL>}<EOL>ver.save()<EOL>return ver<EOL>

Create a new version under this service.


vcl = VCL()<EOL>vcl.conn = self.conn<EOL>vcl.attrs = {<EOL>'<STR_LIT>': self.attrs['<STR_LIT>'],<EOL>'<STR_LIT:version>': self.attrs['<STR_LIT>'],<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT:content>': content,<EOL>}<EOL>vcl.save()<EOL>return vcl<EOL>


stack = [(dst, src)]<EOL>def isdict(o):<EOL><INDENT>return hasattr(o, '<STR_LIT>')<EOL><DEDENT>while stack:<EOL><INDENT>current_dst, current_src = stack.pop()<EOL>for key in current_src:<EOL><INDENT>if key not in current_dst:<EOL><INDENT>current_dst[key] = current_src[key]<EOL><DEDENT>else:<EOL><INDENT>if isdict(current_src[key]) and isdict(current_dst[key]):<EOL><INDENT>stack.append((current_dst[key], current_src[key]))<EOL><DEDENT>else:<EOL><INDENT>current_dst[key] = current_src[key]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return dst<EOL>

Recursively update the destination dict-like object with the source dict-like object.

    Useful for merging options and Bunches together!

    Based on:
    http://code.activestate.com/recipes/499335-recursively-update-a-dictionary-without-hitting-py/#c1


kwargs['<STR_LIT:action>'] = '<STR_LIT>'<EOL>kwargs['<STR_LIT>'] = '<STR_LIT>'<EOL>return _walkWithAction(*patterns, **kwargs)<EOL>

Remove all directories under the current path with the given patterns.


download_cache = ('<STR_LIT>' % options.paved.pip.download_cache) if options.paved.pip.download_cache else '<STR_LIT>'<EOL>shv('<STR_LIT>' % (download_cache, '<STR_LIT:U+0020>'.join(args)))<EOL>

Send the given arguments to `pip install`.


cmd = '<STR_LIT>'<EOL>try:<EOL><INDENT>import django_extensions<EOL>cmd = '<STR_LIT>'<EOL><DEDENT>except ImportError:<EOL><INDENT>info("<STR_LIT>")<EOL><DEDENT>call_manage(cmd)<EOL>

Run the ipython shell. Shorthand for `paver manage shell`.

    Uses `django_extensions <http://pypi.python.org/pypi/django-extensions/0.5>`, if
    available, to provide `shell_plus`.


try:<EOL><INDENT>import south<EOL>cmd = args and '<STR_LIT>' % '<STR_LIT:U+0020>'.join(options.args) or '<STR_LIT>'<EOL>call_manage(cmd)<EOL><DEDENT>except ImportError:<EOL><INDENT>error('<STR_LIT>')<EOL><DEDENT>


<EOL>packages = [x for x in options.setup.packages if '<STR_LIT:.>' not in x]<EOL>sh('<STR_LIT>'.format(param=options.paved.pycheck.pychecker.param, files='<STR_LIT:U+0020>'.join(packages)))<EOL>

check of python programs by pychecker.

    requirements:
     - pychecker_ should be installed.

    options.paved.pycheck.pychecker.param

    .. _pychecker: http://pychecker.sourceforge.net/


<EOL>packages = [x for x in options.setup.packages if '<STR_LIT:.>' not in x]<EOL>sh('<STR_LIT>'.format(param=options.paved.pycheck.findimports.param, files='<STR_LIT:U+0020>'.join(packages)))<EOL>

print python module dependencies by findimports.

    requirements:
     - findimports_ should be installed. ``easy_install findimports``

    options.paved.pycheck.findimports.param

    .. _findimports: http://pypi.python.org/pypi/findimports


sh('<STR_LIT>'.format(param=options.paved.pycheck.nose.param))<EOL>

Run unit tests using nosetests.

    requirements:
     - nose_ should be installed.

    options.paved.pycheck.nose.param

    .. _nose: http://somethingaboutorange.com/mrl/projects/nose/1.0.0/


@task<EOL>@needs('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>def pycheckall():


info("<STR_LIT>", options.paved.clean.patterns)<EOL>for wd in options.paved.clean.dirs:<EOL><INDENT>info("<STR_LIT>", wd)<EOL>for p in options.paved.clean.patterns:<EOL><INDENT>for f in wd.walkfiles(p):<EOL><INDENT>f.remove()<EOL><DEDENT><DEDENT><DEDENT>

Clean up extra files littering the source tree.

    options.paved.clean.dirs: directories to search recursively
    options.paved.clean.patterns: patterns to search for and remove


x = json.dumps(environment.options,<EOL>indent=<NUM_LIT:4>,<EOL>sort_keys=True,<EOL>skipkeys=True,<EOL>cls=MyEncoder)<EOL>print(x)<EOL>

print paver options.

    Prettified by json.
    `long_description` is removed


conn = boto.connect_s3(options.paved.s3.access_id, options.paved.s3.secret)<EOL>try:<EOL><INDENT>bucket = conn.get_bucket(bucket)<EOL><DEDENT>except boto.exception.S3ResponseError:<EOL><INDENT>bucket = conn.create_bucket(bucket)<EOL><DEDENT>return bucket<EOL>

Opens connection to S3 returning bucket and key


def download_s3(bucket_name, file_key, file_path, force=False):

file_path = path(file_path)<EOL>bucket = open_s3(bucket_name)<EOL>file_dir = file_path.dirname()<EOL>file_dir.makedirs()<EOL>s3_key = bucket.get_key(file_key)<EOL>if file_path.exists():<EOL><INDENT>file_data = file_path.bytes()<EOL>file_md5, file_md5_64 = s3_key.get_md5_from_hexdigest(hashlib.md5(file_data).hexdigest())<EOL>try:<EOL><INDENT>s3_md5 = s3_key.etag.replace('<STR_LIT:">', '<STR_LIT>')<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>if s3_md5 == file_md5:<EOL><INDENT>info('<STR_LIT>' % file_path)<EOL>return<EOL><DEDENT>elif not force:<EOL><INDENT>s3_datetime = datetime.datetime(*time.strptime(<EOL>s3_key.last_modified, '<STR_LIT>')[<NUM_LIT:0>:<NUM_LIT:6>])<EOL>local_datetime = datetime.datetime.utcfromtimestamp(file_path.stat().st_mtime)<EOL>if s3_datetime < local_datetime:<EOL><INDENT>info("<STR_LIT>" % (file_key))<EOL>return<EOL><DEDENT><DEDENT><DEDENT><DEDENT>info("<STR_LIT>" % (file_key))<EOL>try:<EOL><INDENT>with open(file_path, '<STR_LIT:w>') as fo:<EOL><INDENT>s3_key.get_contents_to_file(fo)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>error("<STR_LIT>" % e)<EOL>raise<EOL><DEDENT>


sh('<STR_LIT>' % '<STR_LIT:U+0020>'.join(targets), cwd=options.paved.docs.path)<EOL>

Call the Sphinx Makefile with the specified targets.

    `options.paved.docs.path`: the path to the Sphinx folder (where the Makefile resides).


sphinx_make(*options.paved.docs.targets)<EOL>

Make Sphinx docs.

    `options.paved.docs.path`: the path to the Sphinx folder (where the Makefile resides).

    `options.paved.docs.targets`: the Make targets to send to `sphinx_make`. Default is `html`.


assert options.paved.docs.rsync_location, "<STR_LIT>"<EOL>sh('<STR_LIT>' % (path(options.paved.docs.path) / options.paved.docs.build_rel,<EOL>options.paved.docs.rsync_location))<EOL>

Upload the docs to a remote location via rsync.

    `options.paved.docs.rsync_location`: the target location to rsync files to.

    `options.paved.docs.path`: the path to the Sphinx folder (where the Makefile resides).

    `options.paved.docs.build_rel`: the path of the documentation
        build folder, relative to `options.paved.docs.path`.


Send the given arguments to `easy_install`.


messages = self.validate_creators(messages)<EOL>messages = self.validate_created(messages)<EOL>return messages<EOL>

Returns True if the fields are valid according to the SPDX standard.
        Appends user friendly messages to the messages parameter.


licenses_map = {}<EOL>with codecs.open(file_name, '<STR_LIT:rb>', encoding='<STR_LIT:utf-8>') as lics:<EOL><INDENT>licenses = json.load(lics)<EOL>version = licenses['<STR_LIT>'].split('<STR_LIT:.>')<EOL>for lic in licenses['<STR_LIT>']:<EOL><INDENT>if lic.get('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>name = lic['<STR_LIT:name>']<EOL>identifier = lic['<STR_LIT>']<EOL>licenses_map[name] = identifier<EOL>licenses_map[identifier] = name<EOL><DEDENT><DEDENT>return version, licenses_map<EOL>

Return the licenses list version tuple and a mapping of licenses
name->id and id->name loaded from a JSON file
from https://github.com/spdx/license-list-data


return '<STR_LIT>'.format(text) if required else text<EOL>

Add parens around a license expression if `required` is True, otherwise
return `text` unmodified.


messages = self.validate_version(messages)<EOL>messages = self.validate_data_lics(messages)<EOL>messages = self.validate_name(messages)<EOL>messages = self.validate_spdx_id(messages)<EOL>messages = self.validate_namespace(messages)<EOL>messages = self.validate_ext_document_references(messages)<EOL>messages = self.validate_creation_info(messages)<EOL>messages = self.validate_package(messages)<EOL>messages = self.validate_extracted_licenses(messages)<EOL>messages = self.validate_reviews(messages)<EOL>return messages<EOL>

Validate all fields of the document and update the
messages list with user friendly error messages for display.


if identifier in config.LICENSE_MAP.keys():<EOL><INDENT>return cls(config.LICENSE_MAP[identifier], identifier)<EOL><DEDENT>else:<EOL><INDENT>return cls(identifier, identifier)<EOL><DEDENT>

If identifier exists in config.LICENSE_MAP
        the full_name is retrieved from it. Otherwise
        the full_name is the same as the identifier.


if full_name in config.LICENSE_MAP.keys():<EOL><INDENT>return cls(full_name, config.LICENSE_MAP[full_name])<EOL><DEDENT>else:<EOL><INDENT>return cls(full_name, full_name)<EOL><DEDENT>

Returna new License for a full_name. If the full_name exists in
config.LICENSE_MAP the identifier is retrieved from it.
Otherwise the identifier is the same as the full_name.


messages = self.validate_ext_doc_id(messages)<EOL>messages = self.validate_spdx_doc_uri(messages)<EOL>messages = self.validate_checksum(messages)<EOL>return messages<EOL>

Validate all fields of the ExternalDocumentRef class and update the
messages list with user friendly error messages for display.


parser = Parser(Builder(), StandardLogger())<EOL>parser.build()<EOL>with open(infile_name) as infile:<EOL><INDENT>data = infile.read()<EOL>document, error = parser.parse(data)<EOL>if not error:<EOL><INDENT>with open(outfile_name, mode='<STR_LIT:w>') as outfile:<EOL><INDENT>write_document(document, outfile)<EOL><DEDENT>return True<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL>messages = []<EOL>document.validate(messages)<EOL>print('<STR_LIT:\n>'.join(messages))<EOL>return False<EOL><DEDENT><DEDENT>

Convert a SPDX file from tag/value format to RDF format.
Return True on sucess, False otherwise.


doc_node = URIRef('<STR_LIT>')<EOL>self.graph.add((doc_node, RDF.type, self.spdx_namespace.SpdxDocument))<EOL>vers_literal = Literal(str(self.document.version))<EOL>self.graph.add((doc_node, self.spdx_namespace.specVersion, vers_literal))<EOL>data_lics = URIRef(self.document.data_license.url)<EOL>self.graph.add((doc_node, self.spdx_namespace.dataLicense, data_lics))<EOL>doc_name = URIRef(self.document.name)<EOL>self.graph.add((doc_node, self.spdx_namespace.name, doc_name))<EOL>return doc_node<EOL>

Add and return the root document node to graph.


super(Writer, self).__init__(document, out)<EOL>

- document is spdx.document instance that will be written.
- out is a file-like object that will be written to.


for doc_file in self.document.files:<EOL><INDENT>self.add_file_dependencies_helper(doc_file)<EOL><DEDENT>

Add file dependencies to the graph.
Called after all files have been added.


node = BNode()<EOL>type_triple = (node, RDF.type, self.spdx_namespace.ConjunctiveLicenseSet)<EOL>self.graph.add(type_triple)<EOL>licenses = self.licenses_from_tree(conjunction)<EOL>for lic in licenses:<EOL><INDENT>member_triple = (node, self.spdx_namespace.member, lic)<EOL>self.graph.add(member_triple)<EOL><DEDENT>return node<EOL>

Return a node representing a conjunction of licenses.


licenses = list(self.graph.triples((None, self.spdx_namespace.licenseId, lic.identifier)))<EOL>if len(licenses) != <NUM_LIT:0>:<EOL><INDENT>return licenses[<NUM_LIT:0>][<NUM_LIT:0>]  <EOL><DEDENT>else:<EOL><INDENT>license_node = BNode()<EOL>type_triple = (license_node, RDF.type, self.spdx_namespace.ExtractedLicensingInfo)<EOL>self.graph.add(type_triple)<EOL>ident_triple = (license_node, self.spdx_namespace.licenseId, Literal(lic.identifier))<EOL>self.graph.add(ident_triple)<EOL>text_triple = (license_node, self.spdx_namespace.extractedText, Literal(lic.text))<EOL>self.graph.add(text_triple)<EOL>if lic.full_name is not None:<EOL><INDENT>name_triple = (license_node, self.spdx_namespace.licenseName, self.to_special_value(lic.full_name))<EOL>self.graph.add(name_triple)<EOL><DEDENT>for ref in lic.cross_ref:<EOL><INDENT>triple = (license_node, RDFS.seeAlso, URIRef(ref))<EOL>self.graph.add(triple)<EOL><DEDENT>if lic.comment is not None:<EOL><INDENT>comment_triple = (license_node, RDFS.comment, Literal(lic.comment))<EOL>self.graph.add(comment_triple)<EOL><DEDENT>return license_node<EOL><DEDENT>

Handle extracted license.
Return the license node.


review_node = BNode()<EOL>type_triple = (review_node, RDF.type, self.spdx_namespace.Review)<EOL>self.graph.add(type_triple)<EOL>reviewer_node = Literal(review.reviewer.to_value())<EOL>self.graph.add((review_node, self.spdx_namespace.reviewer, reviewer_node))<EOL>reviewed_date_node = Literal(review.review_date_iso_format)<EOL>reviewed_triple = (review_node, self.spdx_namespace.reviewDate, reviewed_date_node)<EOL>self.graph.add(reviewed_triple)<EOL>if review.has_comment:<EOL><INDENT>comment_node = Literal(review.comment)<EOL>comment_triple = (review_node, RDFS.comment, comment_node)<EOL>self.graph.add(comment_triple)<EOL><DEDENT>return review_node<EOL>


nodes = list(self.graph.triples((None, self.spdx_namespace.fileName, Literal(pkg_file.name))))<EOL>if len(nodes) == <NUM_LIT:1>:<EOL><INDENT>return nodes[<NUM_LIT:0>][<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>raise InvalidDocumentError('<STR_LIT>' +<EOL>'<STR_LIT>'.format(pkg_file.name))<EOL><DEDENT>

Return node representing pkg_file
pkg_file should be instance of spdx.file.


return map(self.create_review_node, self.document.reviews)<EOL>


if isinstance(value, utils.NoAssert):<EOL><INDENT>return self.spdx_namespace.noassertion<EOL><DEDENT>elif isinstance(value, utils.SPDXNone):<EOL><INDENT>return self.spdx_namespace.none<EOL><DEDENT>else:<EOL><INDENT>return Literal(value)<EOL><DEDENT>


def write_document(document, out, validate=True):

if validate:<EOL><INDENT>messages = []<EOL>messages = document.validate(messages)<EOL>if messages:<EOL><INDENT>raise InvalidDocumentError(messages)<EOL><DEDENT><DEDENT>writer = Writer(document, out)<EOL>writer.write()<EOL>

Write an SPDX RDF document.
- document - spdx.document instance.
- out - file like object that will be written to.
Optionally `validate` the document before writing and raise
InvalidDocumentError if document.validate returns False.


return map(self.create_file_node, self.document.files)<EOL>


return map(lambda c: Literal(c.to_value()), self.document.creation_info.creators)<EOL>

Return a list of creator nodes.
Note: Does not add anything to the graph.


file_node = URIRef('<STR_LIT>'.format(<EOL>id=str(doc_file.spdx_id)))<EOL>type_triple = (file_node, RDF.type, self.spdx_namespace.File)<EOL>self.graph.add(type_triple)<EOL>name_triple = (file_node, self.spdx_namespace.fileName, Literal(doc_file.name))<EOL>self.graph.add(name_triple)<EOL>if doc_file.has_optional_field('<STR_LIT>'):<EOL><INDENT>comment_triple = (file_node, RDFS.comment, Literal(doc_file.comment))<EOL>self.graph.add(comment_triple)<EOL><DEDENT>if doc_file.has_optional_field('<STR_LIT:type>'):<EOL><INDENT>ftype = self.spdx_namespace[self.FILE_TYPES[doc_file.type]]<EOL>ftype_triple = (file_node, self.spdx_namespace.fileType, ftype)<EOL>self.graph.add(ftype_triple)<EOL><DEDENT>self.graph.add((file_node, self.spdx_namespace.checksum, self.create_checksum_node(doc_file.chk_sum)))<EOL>conc_lic_node = self.license_or_special(doc_file.conc_lics)<EOL>conc_lic_triple = (file_node, self.spdx_namespace.licenseConcluded, conc_lic_node)<EOL>self.graph.add(conc_lic_triple)<EOL>license_info_nodes = map(self.license_or_special, doc_file.licenses_in_file)<EOL>for lic in license_info_nodes:<EOL><INDENT>triple = (file_node, self.spdx_namespace.licenseInfoInFile, lic)<EOL>self.graph.add(triple)<EOL><DEDENT>if doc_file.has_optional_field('<STR_LIT>'):<EOL><INDENT>comment_triple = (file_node, self.spdx_namespace.licenseComments, Literal(doc_file.license_comment))<EOL>self.graph.add(comment_triple)<EOL><DEDENT>cr_text_node = self.to_special_value(doc_file.copyright)<EOL>cr_text_triple = (file_node, self.spdx_namespace.copyrightText, cr_text_node)<EOL>self.graph.add(cr_text_triple)<EOL>if doc_file.has_optional_field('<STR_LIT>'):<EOL><INDENT>notice_triple = (file_node, self.spdx_namespace.noticeText, doc_file.notice)<EOL>self.graph.add(notice_triple)<EOL><DEDENT>contrib_nodes = map(lambda c: Literal(c), doc_file.contributors)<EOL>contrib_triples = [(file_node, self.spdx_namespace.fileContributor, node) for node in contrib_nodes]<EOL>for triple in contrib_triples:<EOL><INDENT>self.graph.add(triple)<EOL><DEDENT>return file_node<EOL>


subj_triples = list(self.graph.triples((None, self.spdx_namespace.fileName, Literal(doc_file.name))))<EOL>if len(subj_triples) != <NUM_LIT:1>:<EOL><INDENT>raise InvalidDocumentError('<STR_LIT>'.format(doc_file.name))<EOL><DEDENT>subject_node = subj_triples[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>for dependency in doc_file.dependencies:<EOL><INDENT>dep_triples = list(self.graph.triples((None, self.spdx_namespace.fileName, Literal(dependency))))<EOL>if len(dep_triples) == <NUM_LIT:1>:<EOL><INDENT>dep_node = dep_triples[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>dep_triple = (subject_node, self.spdx_namespace.fileDependency, dep_node)<EOL>self.graph.add(dep_triple)<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>'.format(doc_file.name, dependency))<EOL><DEDENT><DEDENT>

Handle dependencies for a single file.
- doc_file - instance of spdx.file.File.


out.write('<STR_LIT>')<EOL>write_value('<STR_LIT>', package.name, out)<EOL>if package.has_optional_field('<STR_LIT:version>'):<EOL><INDENT>write_value('<STR_LIT>', package.version, out)<EOL><DEDENT>write_value('<STR_LIT>', package.download_location, out)<EOL>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_text_value('<STR_LIT>', package.summary, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_text_value('<STR_LIT>', package.source_info, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_value('<STR_LIT>', package.file_name, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_value('<STR_LIT>', package.supplier, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_value('<STR_LIT>', package.originator, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_value('<STR_LIT>', package.check_sum.to_tv(), out)<EOL><DEDENT>write_value('<STR_LIT>', format_verif_code(package), out)<EOL>if package.has_optional_field('<STR_LIT:description>'):<EOL><INDENT>write_text_value('<STR_LIT>', package.description, out)<EOL><DEDENT>if isinstance(package.license_declared, (document.LicenseConjunction,<EOL>document.LicenseDisjunction)):<EOL><INDENT>write_value('<STR_LIT>', u'<STR_LIT>'.format(package.license_declared), out)<EOL><DEDENT>else:<EOL><INDENT>write_value('<STR_LIT>', package.license_declared, out)<EOL><DEDENT>if isinstance(package.conc_lics, (document.LicenseConjunction,<EOL>document.LicenseDisjunction)):<EOL><INDENT>write_value('<STR_LIT>', u'<STR_LIT>'.format(package.conc_lics), out)<EOL><DEDENT>else:<EOL><INDENT>write_value('<STR_LIT>', package.conc_lics, out)<EOL><DEDENT>for lics in sorted(package.licenses_from_files):<EOL><INDENT>write_value('<STR_LIT>', lics, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_text_value('<STR_LIT>', package.license_comment, out)<EOL><DEDENT>if isinstance(package.cr_text, six.string_types):<EOL><INDENT>write_text_value('<STR_LIT>', package.cr_text, out)<EOL><DEDENT>else:<EOL><INDENT>write_value('<STR_LIT>', package.cr_text, out)<EOL><DEDENT>if package.has_optional_field('<STR_LIT>'):<EOL><INDENT>write_value('<STR_LIT>', package.homepage, out)<EOL><DEDENT>for spdx_file in sorted(package.files):<EOL><INDENT>write_separators(out)<EOL>write_file(spdx_file, out)<EOL><DEDENT>


write_value('<STR_LIT>', lics.identifier, out)<EOL>if lics.full_name is not None:<EOL><INDENT>write_value('<STR_LIT>', lics.full_name, out)<EOL><DEDENT>if lics.comment is not None:<EOL><INDENT>write_text_value('<STR_LIT>', lics.comment, out)<EOL><DEDENT>for xref in sorted(lics.cross_ref):<EOL><INDENT>write_value('<STR_LIT>', xref, out)<EOL><DEDENT>write_text_value('<STR_LIT>', lics.text, out)<EOL>


out.write('<STR_LIT>')<EOL>write_value('<STR_LIT>', review.reviewer, out)<EOL>write_value('<STR_LIT>', review.review_date_iso_format, out)<EOL>if review.has_comment:<EOL><INDENT>write_text_value('<STR_LIT>', review.comment, out)<EOL><DEDENT>

Write the fields of a single review to out.


out.write('<STR_LIT>')<EOL>write_value('<STR_LIT>', annotation.annotator, out)<EOL>write_value('<STR_LIT>', annotation.annotation_date_iso_format, out)<EOL>if annotation.has_comment:<EOL><INDENT>write_text_value('<STR_LIT>', annotation.comment, out)<EOL><DEDENT>write_value('<STR_LIT>', annotation.annotation_type, out)<EOL>write_value('<STR_LIT>', annotation.spdx_id, out)<EOL>

Write the fields of a single annotation to out.


symbol = '<STR_LIT>'.format(symbol)<EOL>artifact = getattr(self, symbol)<EOL>artifact.append(value)<EOL>

Add value as artifact_of_project{symbol}.


FIELDS = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT:version>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT:description>'<EOL>]<EOL>messages = self.validate_str_fields(FIELDS, True, messages)<EOL>return messages<EOL>

Fields marked as optional and of type string in class
        docstring must be of a type that provides __str__ method.


def validate_str_fields(self, fields, optional, messages):

for field_str in fields:<EOL><INDENT>field = getattr(self, field_str)<EOL>if field is not None:<EOL><INDENT>attr = getattr(field, '<STR_LIT>', None)<EOL>if not callable(attr):<EOL><INDENT>messages = messages + [<EOL>'<STR_LIT>'.format(field)<EOL>]<EOL><DEDENT><DEDENT>elif not optional:<EOL><INDENT>messages = messages + [<EOL>'<STR_LIT>'.format(field_str)<EOL>]<EOL><DEDENT><DEDENT>return messages<EOL>

Helper for validate_mandatory_str_field and
        validate_optional_str_fields


def set_file_license_comment(self, doc, text):

if self.has_package(doc) and self.has_file(doc):<EOL><INDENT>if not self.file_license_comment_set:<EOL><INDENT>self.file_license_comment_set = True<EOL>self.file(doc).license_comment = text<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>raise CardinalityError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise OrderError('<STR_LIT>')<EOL><DEDENT>

Raises OrderError if no package or file defined.
Raises CardinalityError if more than one per file.


<EOL>self.doc_version_set = False<EOL>self.doc_comment_set = False<EOL>self.doc_namespace_set = False<EOL>self.doc_data_lics_set = False<EOL>self.doc_name_set = False<EOL>self.doc_spdx_id_set = False<EOL>

Reset the internal state to allow building new document


def set_pkg_source_info(self, doc, text):

self.assert_package_exists()<EOL>if not self.package_source_info_set:<EOL><INDENT>self.package_source_info_set = True<EOL>doc.package.source_info = text<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>raise CardinalityError('<STR_LIT>')<EOL><DEDENT>

Sets the package's source information, if not already set.
        text - Free form text.
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.


def set_doc_comment(self, doc, comment):

if not self.doc_comment_set:<EOL><INDENT>self.doc_comment_set = True<EOL>doc.comment = comment<EOL><DEDENT>else:<EOL><INDENT>raise CardinalityError('<STR_LIT>')<EOL><DEDENT>

Sets document comment, Raises CardinalityError if
        comment already set.


def set_pkg_chk_sum(self, doc, chk_sum):

self.assert_package_exists()<EOL>if not self.package_chk_sum_set:<EOL><INDENT>self.package_chk_sum_set = True<EOL>doc.package.check_sum = checksum.Algorithm('<STR_LIT>', chk_sum)<EOL><DEDENT>else:<EOL><INDENT>raise CardinalityError('<STR_LIT>')<EOL><DEDENT>

Sets the package check sum, if not already set.
        chk_sum - A string
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.


self.assert_package_exists()<EOL>if not self.package_cr_text_set:<EOL><INDENT>self.package_cr_text_set = True<EOL>doc.package.cr_text = text<EOL><DEDENT>else:<EOL><INDENT>raise CardinalityError('<STR_LIT>')<EOL><DEDENT>

Sets the package's license comment.
        Raises OrderError if no package previously defined.
        Raises CardinalityError if already set.