def _validate_tutorial_link(self, tutorial_link, field, value):

pass<EOL>


def _validate_examples(self, examples, field, value):

pass<EOL>


def validate_duration(self, field, duration):

DURATION_RE = r'<STR_LIT>'<EOL>if not re.match(DURATION_RE, duration):<EOL><INDENT>self._error(field, '<STR_LIT>')<EOL><DEDENT>

2h
2h5m
5m
180
1h4m3
:param duration:
:return:


def _validator_load_scheme(self, field, value):

<EOL>if self.document['<STR_LIT>'] in '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>PRIMARY_RE = r'<STR_LIT>'<EOL>N_OF_ARGS = {<EOL>'<STR_LIT>': <NUM_LIT:4>,<EOL>'<STR_LIT>': <NUM_LIT:3>,<EOL>'<STR_LIT>': <NUM_LIT:2>,<EOL>}<EOL>matches = re.findall(PRIMARY_RE, value)<EOL>if len(matches) == <NUM_LIT:0>:<EOL><INDENT>self._error(field, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>for match in matches:<EOL><INDENT>curve, params_str = match<EOL>params = [v.strip() for v in params_str.split('<STR_LIT:U+002C>')]<EOL>if not len(params) == N_OF_ARGS[curve]:<EOL><INDENT>self._error(field, '<STR_LIT>'.format(curve,<EOL>N_OF_ARGS[curve],<EOL>len(params)))<EOL><DEDENT>for param in params[:-<NUM_LIT:1>]:<EOL><INDENT>if not self.is_number(param):<EOL><INDENT>self._error(field, '<STR_LIT>'.format(param))<EOL><DEDENT><DEDENT>self.validate_duration(field, params[-<NUM_LIT:1>])<EOL><DEDENT><DEDENT>


self._errors = None<EOL>if not isinstance(configs, list):<EOL><INDENT>configs = [configs]<EOL><DEDENT>self.raw_config_dict = self.__load_multiple(<EOL>[config for config in configs if config is not None])<EOL>if self.raw_config_dict.get(core_section) is None:<EOL><INDENT>self.raw_config_dict[core_section] = {}<EOL><DEDENT>self.with_dynamic_options = with_dynamic_options<EOL>self.CORE_SECTION = core_section<EOL>self._validated = None<EOL>self._plugins = None<EOL>self.ERROR_OUTPUT = error_output<EOL>self.BASE_SCHEMA = load_yaml_schema(pkg_resources.resource_filename('<STR_LIT>', '<STR_LIT>'))<EOL>self.PLUGINS_SCHEMA = load_yaml_schema(pkg_resources.resource_filename('<STR_LIT>', '<STR_LIT>'))<EOL>

:param configs: list of configs dicts
:param with_dynamic_options: insert uuid, pid, and other DYNAMIC_OPTIONS
:param core_section: name of core section in config
:param error_output: file to output error messages


return [<EOL>(<EOL>plugin_name,<EOL>plugin['<STR_LIT>'],<EOL>plugin) for plugin_name,<EOL>plugin in self.raw_config_dict.items() if (<EOL>plugin_name not in self.BASE_SCHEMA.keys()) and isinstance(<EOL>plugin,<EOL>dict) and plugin.get('<STR_LIT>')]<EOL>

:returns: [(plugin_name, plugin_package, plugin_config), ...]
:rtype: list of tuple


def __init__(self, validated, base_schema):

self.validated = validated<EOL>self.base_schema = base_schema<EOL>self._plugins = None<EOL>


if not self._plugins:<EOL><INDENT>self._plugins = [<EOL>(plugin_name,<EOL>plugin_cfg['<STR_LIT>'],<EOL>plugin_cfg) for plugin_name, plugin_cfg in self.validated.items() if (<EOL>plugin_name not in self.base_schema.keys()) and plugin_cfg['<STR_LIT>']]<EOL><DEDENT>return self._plugins<EOL>


def render_body(renderer, option_kwargs, exclude_keys, special_keys=None):

common_formatters = {<EOL>EXAMPLES: lambda examples: renderer.def_list({renderer.mono(example): annotation for example, annotation in examples.items()})<EOL>}<EOL>def default_fmt(x):<EOL><INDENT>return x<EOL><DEDENT>special_keys = special_keys or {}<EOL>special_part = '<STR_LIT:\n>'.join([special_handler(renderer, option_kwargs[special_key])<EOL>for special_key, special_handler in special_keys.items()<EOL>if special_key in option_kwargs])<EOL>common_part = renderer.field_list({k: common_formatters.get(k, default_fmt)(v) for k, v in option_kwargs.items()<EOL>if k not in exclude_keys + special_keys.keys()})<EOL>return '<STR_LIT:\n>'.join([_ for _ in [common_part, special_part] if _])<EOL>

:type option_kwargs: dict
:type exclude_keys: list
:type special_keys: dict


return OptionFormatter(option_schema).formatter<EOL>


def format_schema(schema, renderer, title=None):

body = '<STR_LIT>'.join(<EOL>sorted([format_option({option_name: option_schema}, renderer) for option_name, option_schema in schema.items()]))<EOL>if title:<EOL><INDENT>title = renderer.title(title)<EOL>return title + '<STR_LIT>' + body<EOL><DEDENT>else:<EOL><INDENT>return body<EOL><DEDENT>

:param dict schema: Cerberus config schema
:type renderer: RSTRenderer


def __init__(self, text, tab_replacement='<STR_LIT:U+0020>', ending='<STR_LIT>'):

self.text = str(text).replace('<STR_LIT:\t>', tab_replacement)<EOL>self.lines = self.text.splitlines()<EOL>self.width = max([len(line) for line in self.lines] + [<NUM_LIT:0>])<EOL>self.padded_width = self.width + <NUM_LIT:2><EOL>self.height = len(self.lines)<EOL>


HEADER = '<STR_LIT>'<EOL>cnt = len(blocks)<EOL>if cnt < <NUM_LIT:2>:<EOL><INDENT>return blocks[<NUM_LIT:0>] if blocks else '<STR_LIT>'<EOL><DEDENT>width = max((len(HEADER), sum([c.padded_width for c in blocks]))) + (cnt + <NUM_LIT:1>)<EOL>height = max([c.height for c in blocks])<EOL>top_bar = '<STR_LIT>'.format('<STR_LIT:->' * (width - <NUM_LIT:2>))<EOL>header_bar = '<STR_LIT>'.format('<STR_LIT:+>'.join(['<STR_LIT:=>' * c.padded_width for c in blocks]))<EOL>bottom_bar = '<STR_LIT>'.format('<STR_LIT:+>'.join(['<STR_LIT:->' * c.padded_width for c in blocks]))<EOL>header = '<STR_LIT>'.format(HEADER.center(width - <NUM_LIT:2>))<EOL>body = '<STR_LIT:\n>'.join(<EOL>['<STR_LIT>'.format('<STR_LIT>'.join([c.get_line_justified(i) for c in blocks])) for i in range(height)])<EOL>return '<STR_LIT:\n>'.join([top_bar,<EOL>header,<EOL>header_bar,<EOL>body,<EOL>bottom_bar])<EOL>


return '<STR_LIT:\n>'.join(['<STR_LIT>'.format(line) for line in block.lines])<EOL>


return '<STR_LIT:\n>'.join(['<STR_LIT>'.format(line) for line in content.splitlines()])<EOL>


prepared_content = content.strip().replace('<STR_LIT:\n>', new_line_replacement).replace('<STR_LIT:\t>', tab_replacement)<EOL>return u'<STR_LIT>'.format(prepared_content, '<STR_LIT:=>' * len(prepared_content))<EOL>

Underlines content with '='. New lines and tabs will be replaced
:param str content:
:param str new_line_replacement:
:param str tab_replacement:
:return: unicode


return TextBlock('<STR_LIT:\n>'.join([cls._list_item(block) for block in blocks]))<EOL>

:type blocks: list of TextBlock
:rtype: TextBlock


return '<STR_LIT>' + '<STR_LIT>'.join(block.lines)<EOL>


@staticmethod<EOL><INDENT>def field_list(items, sort=True, newlines=True):<DEDENT>

def format_value(value):<EOL><INDENT>if isinstance(value, (int, bool, NoneType)):<EOL><INDENT>return format_value(str(value))<EOL><DEDENT>if isinstance(value, (str, unicode)):<EOL><INDENT>return '<STR_LIT>'.join(value.splitlines())<EOL><DEDENT>elif isinstance(value, TextBlock):<EOL><INDENT>return '<STR_LIT>'.join(value.lines)<EOL><DEDENT>elif isinstance(value, dict):<EOL><INDENT>return '<STR_LIT>'.join(RSTRenderer.field_list(value, sort, newlines).splitlines())<EOL><DEDENT>elif isinstance(value, list):<EOL><INDENT>return '<STR_LIT>'.join(RSTRenderer.bullet_list([TextBlock(item) for item in value]).lines)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(type(value), value))<EOL><DEDENT><DEDENT>sort = sorted if sort else lambda x: x<EOL>template = '<STR_LIT>' if newlines else '<STR_LIT>'<EOL>return '<STR_LIT:\n>' + '<STR_LIT:\n>'.join([template.format(k.replace('<STR_LIT:\n>', '<STR_LIT:U+0020>'),<EOL>format_value(v).strip())<EOL>for k, v in sort(items.items())]) if items else '<STR_LIT>'<EOL>

:param bool newlines: add newlines between names and values
:param bool sort: sort items alphabetically by key
:type items: dict
:rtype: TextBlock


return content.replace('<STR_LIT:->', r'<STR_LIT>')<EOL>


self.option_name, self.option_kwargs = option_schema.items()[<NUM_LIT:0>]<EOL>self.formatter = self.__guess_formatter()<EOL>


self.generator = generator<EOL>self.listeners = []  <EOL>self.results = q.Queue()<EOL>self.stats_results = q.Queue()<EOL>self.data_cache = {}<EOL>self.stat_cache = {}<EOL>self.reader = None<EOL>self.stats_reader = None<EOL>self.drain = None<EOL>self.stats_drain = None<EOL>


data = get_nowait_from_queue(self.results)<EOL>stats = get_nowait_from_queue(self.stats_results)<EOL>logger.debug("<STR_LIT>" % [d.get('<STR_LIT>') for d in data])<EOL>logger.debug("<STR_LIT>" % [d.get('<STR_LIT>') for d in stats])<EOL>for item in data:<EOL><INDENT>ts = item['<STR_LIT>']<EOL>if ts in self.stat_cache:<EOL><INDENT>data_item = item<EOL>stat_item = self.stat_cache.pop(ts)<EOL>self.__notify_listeners(data_item, stat_item)<EOL><DEDENT>else:<EOL><INDENT>self.data_cache[ts] = item<EOL><DEDENT><DEDENT>for item in stats:<EOL><INDENT>ts = item['<STR_LIT>']<EOL>if ts in self.data_cache:<EOL><INDENT>data_item = self.data_cache.pop(ts)<EOL>stat_item = item<EOL>self.__notify_listeners(data_item, stat_item)<EOL><DEDENT>else:<EOL><INDENT>self.stat_cache[ts] = item<EOL><DEDENT><DEDENT>if end and len(self.data_cache) > <NUM_LIT:0>:<EOL><INDENT>logger.info('<STR_LIT>')<EOL>for ts, data_item in sorted(self.data_cache.items(), key=lambda i: i[<NUM_LIT:0>]):<EOL><INDENT>logger.info(ts)<EOL>self.__notify_listeners(data_item, StatsReader.stats_item(ts, <NUM_LIT:0>, <NUM_LIT:0>))<EOL><DEDENT><DEDENT>

Collect data, cache it and send to listeners


def __notify_listeners(self, data, stats):

for listener in self.listeners:<EOL><INDENT>listener.on_aggregated_data(data, stats)<EOL><DEDENT>

notify all listeners about aggregate data and stats


def log_stdout_stderr(log, stdout, stderr, comment="<STR_LIT>"):

readable = select.select([stdout], [], [], <NUM_LIT:0>)[<NUM_LIT:0>]<EOL>if stderr:<EOL><INDENT>exceptional = select.select([stderr], [], [], <NUM_LIT:0>)[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>exceptional = []<EOL><DEDENT>log.debug("<STR_LIT>", readable, exceptional)<EOL>for handle in readable:<EOL><INDENT>line = handle.read()<EOL>readable.remove(handle)<EOL>if line:<EOL><INDENT>log.debug("<STR_LIT>", comment, line.strip())<EOL><DEDENT><DEDENT>for handle in exceptional:<EOL><INDENT>line = handle.read()<EOL>exceptional.remove(handle)<EOL>if line:<EOL><INDENT>log.warn("<STR_LIT>", comment, line.strip())<EOL><DEDENT><DEDENT>

This function polls stdout and stderr streams and writes their contents
to log


return expand_time(str_time, '<STR_LIT>', <NUM_LIT:1000>)<EOL>


return expand_time(str_time, '<STR_LIT:s>', <NUM_LIT:1>)<EOL>


def expand_time(str_time, default_unit='<STR_LIT:s>', multiplier=<NUM_LIT:1>):

parser = re.compile(r'<STR_LIT>')<EOL>parts = parser.findall(str_time)<EOL>result = <NUM_LIT:0.0><EOL>for value, unit in parts:<EOL><INDENT>value = int(value)<EOL>unit = unit.lower()<EOL>if unit == '<STR_LIT>':<EOL><INDENT>unit = default_unit<EOL><DEDENT>if unit == '<STR_LIT>':<EOL><INDENT>result += value * <NUM_LIT><EOL>continue<EOL><DEDENT>elif unit == '<STR_LIT:s>':<EOL><INDENT>result += value<EOL>continue<EOL><DEDENT>elif unit == '<STR_LIT:m>':<EOL><INDENT>result += value * <NUM_LIT><EOL>continue<EOL><DEDENT>elif unit == '<STR_LIT:h>':<EOL><INDENT>result += value * <NUM_LIT> * <NUM_LIT><EOL>continue<EOL><DEDENT>elif unit == '<STR_LIT:d>':<EOL><INDENT>result += value * <NUM_LIT> * <NUM_LIT> * <NUM_LIT><EOL>continue<EOL><DEDENT>elif unit == '<STR_LIT:w>':<EOL><INDENT>result += value * <NUM_LIT> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT:7><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>" % (unit, str_time))<EOL><DEDENT><DEDENT>return int(result * multiplier)<EOL>


if pid < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>try:<EOL><INDENT>os.kill(pid, <NUM_LIT:0>)<EOL><DEDENT>except OSError as exc:<EOL><INDENT>logging.debug("<STR_LIT>", exc.errno, exc)<EOL>return exc.errno == errno.EPERM<EOL><DEDENT>else:<EOL><INDENT>p = psutil.Process(pid)<EOL>return p.status != psutil.STATUS_ZOMBIE<EOL><DEDENT>

Check whether pid exists in the current process table.


patt = re.compile(r'<STR_LIT>')<EOL>if patt.search(string):<EOL><INDENT>quoted_item = patt.search(string).group()<EOL>newstring = patt.sub('<STR_LIT>', string)<EOL>return newstring.split() + [quoted_item]<EOL><DEDENT>else:<EOL><INDENT>return string.split()<EOL><DEDENT>

>>> string = 'apple orange "banana tree" green'
>>> splitstring(string)
['apple', 'orange', 'green', '"banana tree"']


return itertools.izip(lst[::<NUM_LIT:2>], lst[<NUM_LIT:1>::<NUM_LIT:2>])<EOL>


def read_with_lock(self, pos, _len=None):

self.wait_lock()<EOL>try:<EOL><INDENT>self._opened_file.seek(pos)<EOL>result = self._opened_file.read(_len) if _len is not None else self._opened_file.readline()<EOL>stop_pos = self._opened_file.tell()<EOL><DEDENT>finally:<EOL><INDENT>self.unlock()<EOL><DEDENT>if not result and self.stop.is_set():<EOL><INDENT>result = None<EOL><DEDENT>return result, stop_pos<EOL>

Reads {_len} characters if _len is not None else reads line
:param pos: start reading position
:param _len: number of characters to read
:rtype: (string, int)


def __init__(self, multireader, cache_size):

self.multireader = multireader<EOL>self.cache_size = cache_size<EOL>self._cursor = <NUM_LIT:0><EOL>


raise TypeError("<STR_LIT>")<EOL>

Get dictionary key for plugin,
        should point to __file__ magic constant


super(AbstractPlugin, self).__init__()<EOL>self._cleanup_actions = []<EOL>self.log = logging.getLogger(__name__)<EOL>self.core = core<EOL>self.cfg = cfg<EOL>self.cfg_section_name = name<EOL>self.interrupted = self.core.interrupted<EOL>

:param name:
:type core: yandextank.core.TankCore
:type cfg: dict


pass<EOL>

A stage to read config values and instantiate objects


assert callable(action)<EOL>self._cleanup_actions.append(action)<EOL>


return retcode<EOL>


def get_option(self, option_name, default_value=None):

return self.cfg.get(option_name, default_value)<EOL>

Wrapper to get option from plugins' section


return []<EOL>

returns array containing known options for plugin


self.log.debug(<EOL>"<STR_LIT>", self.__class__.__name__, key, value)<EOL>self.core.publish(self.__class__.__name__, key, value)<EOL>


pass<EOL>

Release allocated resources here.
Warning: don't do any logic or potentially dangerous operations


raise NotImplementedError("<STR_LIT>")<EOL>

Notification about new monitoring data lines


def on_aggregated_data(self, data, stats):

notification about new aggregated data and stats

data contains aggregated metrics and stats contain non-aggregated
metrics from gun (like instances count, for example)

data and stats are cached and synchronized by timestamp. Stat items
are holded until corresponding data item is received and vice versa.


return <NUM_LIT:0><EOL>


total = <NUM_LIT:0><EOL>for code, count in codes_dict.items():<EOL><INDENT>if codes_regex.match(str(code)):<EOL><INDENT>total += count<EOL><DEDENT><DEDENT>return total<EOL>


notification about aggregate data goes here


long explanation to show after test stop


returns dict with all criterion parameters


return self.explain(), <NUM_LIT:0><EOL>

short explanation to display in right panel


returns string that used as config name for criterion


return self.Info(**self.DEFAULT_INFO)<EOL>


pass<EOL>


return self._session.query(*columns)<EOL>

[session.query]

        Returns:
            [Query] -- [sqlalchemy Query cls]


if isinstance(t_obj, Iterable):<EOL><INDENT>self._session.add_all(t_obj)<EOL><DEDENT>else:<EOL><INDENT>self._session.add(t_obj)<EOL><DEDENT>

[update table]

        Arguments:
            t_obj {[objs of DeclarativeMeta]} -- [update the table]


def insert(self, table, insert_obj, ignore=True):

if isinstance(insert_obj, pd.DataFrame):<EOL><INDENT>if insert_obj.empty:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>insert_obj = insert_obj.to_dict(orient='<STR_LIT>')<EOL><DEDENT>elif not isinstance(insert_obj, list):<EOL><INDENT>raise ValueError(<EOL>f"<STR_LIT>")<EOL><DEDENT>ignore_str = '<STR_LIT>' if ignore else '<STR_LIT>'<EOL>return self._session.execute(<EOL>table.__table__.insert().prefix_with(ignore_str), insert_obj)<EOL>

[insert bulk data]

        Arguments:
            table {[DeclarativeMeta cls]} -- [reflection of table]
            insert_obj {[pd.DataFrame or list of dicts]} -- [insert_obj]

        Keyword Arguments:
            ignore {bool} -- [wether ignore exception or not] (default: {True})

        Raises:
            ValueError -- [f"The {reprlib.repr(insert_obj)} must be list of dicts type!"]

        Returns:
            [type] -- [description]


return pd.read_sql(sql=self.statement, con=self.session.bind, **kwargs)<EOL>

[pandas.read_sql]

        Arguments:
            Query {[type]} -- [description]

        Returns:
            [pd.DataFrame or generate] -- [description]


result = first_name()<EOL>if with_num:<EOL><INDENT>result += str(random.randint(<NUM_LIT>, <NUM_LIT>))<EOL><DEDENT>return result.lower()<EOL>

Random user name.

Basically it's lowercased result of
:py:func:`~forgery_py.forgery.name.first_name()` with a number appended
if `with_num`.


return random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>


result = random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>result += '<STR_LIT:.>' + top_level_domain()<EOL>return result.lower()<EOL>

Random domain name.

Lowercased result of :py:func:`~forgery_py.forgery.name.company_name()`
plus :py:func:`~top_level_domain()`.


if not user:<EOL><INDENT>user = user_name()<EOL><DEDENT>else:<EOL><INDENT>user = user.strip().replace('<STR_LIT:U+0020>', '<STR_LIT:_>').lower()<EOL><DEDENT>return user + '<STR_LIT:@>' + domain_name()<EOL>

Random e-mail address in a hopefully imaginary domain.

If `user` is ``None`` :py:func:`~user_name()` will be used. Otherwise it
will be lowercased and will have spaces replaced with ``_``.

Domain name is created using :py:func:`~domain_name()`.


return '<STR_LIT:.>'.join([str(random.randint(<NUM_LIT:0>, <NUM_LIT:255>)) for i in xrange(<NUM_LIT:0>, <NUM_LIT:4>)])<EOL>


return words(quantity=<NUM_LIT:1>)<EOL>


global _words<EOL>if not _words:<EOL><INDENT>_words = '<STR_LIT:U+0020>'.join(get_dictionary('<STR_LIT>')).lower().replace('<STR_LIT:\n>', '<STR_LIT>')<EOL>_words = re.sub(r'<STR_LIT>', '<STR_LIT>', _words)<EOL>_words = _words.split('<STR_LIT:U+0020>')<EOL><DEDENT>result = random.sample(_words, quantity)<EOL>if as_list:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:U+0020>'.join(result)<EOL><DEDENT>


result = words(quantity=words_quantity)<EOL>result += random.choice('<STR_LIT>')<EOL>return result.capitalize()<EOL>

Random sentence to be used as e.g. an e-mail subject.


return sentences(quantity=<NUM_LIT:1>)<EOL>


result = [sntc.strip() for sntc in<EOL>random.sample(get_dictionary('<STR_LIT>'), quantity)]<EOL>if as_list:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:U+0020>'.join(result)<EOL><DEDENT>


def paragraph(separator='<STR_LIT>', wrap_start='<STR_LIT>', wrap_end='<STR_LIT>',<EOL>html=False, sentences_quantity=<NUM_LIT:3>):

return paragraphs(quantity=<NUM_LIT:1>, separator=separator, wrap_start=wrap_start,<EOL>wrap_end=wrap_end, html=html,<EOL>sentences_quantity=sentences_quantity)<EOL>


def paragraphs(quantity=<NUM_LIT:2>, separator='<STR_LIT>', wrap_start='<STR_LIT>', wrap_end='<STR_LIT>',<EOL>html=False, sentences_quantity=<NUM_LIT:3>, as_list=False):

if html:<EOL><INDENT>wrap_start = '<STR_LIT>'<EOL>wrap_end = '<STR_LIT>'<EOL>separator = '<STR_LIT>'<EOL><DEDENT>result = []<EOL>for i in xrange(<NUM_LIT:0>, quantity):<EOL><INDENT>result.append(wrap_start + sentences(sentences_quantity) + wrap_end)<EOL><DEDENT>if as_list:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>return separator.join(result)<EOL><DEDENT>


_dict = get_dictionary('<STR_LIT>')<EOL>_dict += get_dictionary('<STR_LIT>')<EOL>return random.choice(_dict).strip()<EOL>


return first_name() + '<STR_LIT:U+0020>' + last_name()<EOL>

Random full name. Equivalent of ``first_name() + ' ' + last_name()``.


result = random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>result = result.replace('<STR_LIT>', job_title_suffix())<EOL>return result<EOL>


Random location name, e.g. ``MI6 Headquarters``.


if abbr:<EOL><INDENT>return random.choice(DAYS_ABBR)<EOL><DEDENT>else:<EOL><INDENT>return random.choice(DAYS)<EOL><DEDENT>

Random (abbreviated if `abbr`) day of week name.


if numerical:<EOL><INDENT>return random.randint(<NUM_LIT:1>, <NUM_LIT:12>)<EOL><DEDENT>else:<EOL><INDENT>if abbr:<EOL><INDENT>return random.choice(MONTHS_ABBR)<EOL><DEDENT>else:<EOL><INDENT>return random.choice(MONTHS)<EOL><DEDENT><DEDENT>

Random (abbreviated if `abbr`) month name or month number if
`numerical`.


def year(past=False, min_delta=<NUM_LIT:0>, max_delta=<NUM_LIT:20>):

return datetime.date.today().year + _delta(past, min_delta, max_delta)<EOL>


return random.randint(<NUM_LIT:1>, month_length)<EOL>

Random day number in a `month_length` days long month.


def date(past=False, min_delta=<NUM_LIT:0>, max_delta=<NUM_LIT:20>):

timedelta = datetime.timedelta(days=_delta(past, min_delta, max_delta))<EOL>return datetime.date.today() + timedelta<EOL>

Random `datetime.date` object. Delta args are days.


return random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>

Random currency description, e.g. `United Kingdom Pounds`.


return '<STR_LIT>'.join(random.sample(HEX_DIGITS, <NUM_LIT:6>))<EOL>


return '<STR_LIT>'.join(random.sample(HEX_DIGITS, <NUM_LIT:3>))<EOL>


def text(length=None, at_least=<NUM_LIT:10>, at_most=<NUM_LIT:15>, lowercase=True,<EOL>uppercase=True, digits=True, spaces=True, punctuation=False):

base_string = '<STR_LIT>'<EOL>if lowercase:<EOL><INDENT>base_string += string.ascii_lowercase<EOL><DEDENT>if uppercase:<EOL><INDENT>base_string += string.ascii_uppercase<EOL><DEDENT>if digits:<EOL><INDENT>base_string += string.digits<EOL><DEDENT>if spaces:<EOL><INDENT>base_string += '<STR_LIT:U+0020>'<EOL><DEDENT>if punctuation:<EOL><INDENT>base_string += string.punctuation<EOL><DEDENT>if len(base_string) == <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if not length:<EOL><INDENT>length = random.randint(at_least, at_most)<EOL><DEDENT>result = '<STR_LIT>'<EOL>for i in xrange(<NUM_LIT:0>, length):<EOL><INDENT>result += random.choice(base_string)<EOL><DEDENT>return result<EOL>

Random text.

If `length` is present the text will be exactly this chars long. Else the
text will be something between `at_least` and `at_most` chars long.


return gender()[<NUM_LIT:0>:<NUM_LIT:1>]<EOL>


length = int(random.choice(string.digits[<NUM_LIT:1>:<NUM_LIT:6>]))<EOL>return '<STR_LIT>'.join(random.sample(string.digits, length))<EOL>


return '<STR_LIT>' % (street_number(), street_name(), street_suffix())<EOL>

Random street address. Equivalent of ``street_number() + ' ' +
street_name() + ' ' + street_suffix()``.


format = '<STR_LIT>'<EOL>if random.random() >= <NUM_LIT:0.5>:<EOL><INDENT>format = '<STR_LIT>'<EOL><DEDENT>result = '<STR_LIT>'<EOL>for item in format:<EOL><INDENT>if item == '<STR_LIT:#>':<EOL><INDENT>result += str(random.randint(<NUM_LIT:0>, <NUM_LIT:9>))<EOL><DEDENT>else:<EOL><INDENT>result += item<EOL><DEDENT><DEDENT>return result<EOL>

Random ZIP code, either in `#####` or `#####-####` format.


format = '<STR_LIT>'<EOL>result = '<STR_LIT>'<EOL>for item in format:<EOL><INDENT>if item == '<STR_LIT:#>':<EOL><INDENT>result += str(random.randint(<NUM_LIT:0>, <NUM_LIT:9>))<EOL><DEDENT>else:<EOL><INDENT>result += item<EOL><DEDENT><DEDENT>return result<EOL>

Random phone number in `#-(###)###-####` format.


global dictionaries_cache<EOL>if dict_name not in dictionaries_cache:<EOL><INDENT>try:<EOL><INDENT>dictionary_file = codecs.open(<EOL>join(DICTIONARIES_PATH, dict_name), '<STR_LIT:r>', '<STR_LIT:utf-8>'<EOL>)<EOL><DEDENT>except IOError:<EOL><INDENT>None<EOL><DEDENT>else:<EOL><INDENT>dictionaries_cache[dict_name] = dictionary_file.readlines()<EOL>dictionary_file.close()<EOL><DEDENT><DEDENT>return dictionaries_cache[dict_name]<EOL>

Load a dictionary file ``dict_name`` (if it's not cached) and return its
contents as an array of strings.


tree = parser.parse(input_string)<EOL>visitor = ChatlVisitor(prefix)<EOL>visit_parse_tree(tree, visitor)<EOL>return visitor.parsed<EOL>

Parses the given DSL string and returns parsed results.

    Args:
      input_string (str): DSL string
      prefix (str): Optional prefix to add to every element name, useful to namespace things

    Returns:
      dict: Parsed content


for k, v in u.items():<EOL><INDENT>if isinstance(v, Mapping):<EOL><INDENT>d[k] = deep_update(d.get(k, {}), v)<EOL><DEDENT>elif isinstance(v, list):<EOL><INDENT>existing_elements = d.get(k, [])<EOL>d[k] = existing_elements + [ele for ele in v if ele not in existing_elements]<EOL><DEDENT>else:<EOL><INDENT>d[k] = v<EOL><DEDENT><DEDENT>return d<EOL>

Deeply updates a dictionary. List values are concatenated.

    Args:
      d (dict): First dictionary which will be updated
      u (dict): Second dictionary use to extend the first one

    Returns:
      dict: The merge dictionary


return self._rendition_set_config.get(<EOL>value['<STR_LIT>']<EOL>)<EOL>

Return the rendition associated with `value`.


context = super(RenditionAwareStructBlock, self).get_context(value)<EOL>context['<STR_LIT>'] = self.rendition.image_rendition or '<STR_LIT>'<EOL>return context<EOL>

Ensure `image_rendition` is added to the global context.


for app in settings.INSTALLED_APPS:<EOL><INDENT>mod = import_module(app)<EOL>try:<EOL><INDENT>before_import_block_registry = copy.copy(<EOL>block_registry._registry<EOL>)<EOL>import_module('<STR_LIT>'.format(app))<EOL><DEDENT>except:<EOL><INDENT>block_registry._registry = before_import_block_registry<EOL>if module_has_submodule(mod, '<STR_LIT>'):<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>

Auto-discover INSTALLED_APPS registered_blocks.py modules and fail
silently when not present. This forces an import on them thereby
registering their blocks.

This is a near 1-to-1 copy of how django's admin application registers
models.


def _verify_block(self, block_type, block):

if block_type in self._registry:<EOL><INDENT>raise AlreadyRegistered(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>block_type<EOL>)<EOL>)<EOL><DEDENT>if not isinstance(block, Block):<EOL><INDENT>raise InvalidBlock(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(block_type)<EOL>)<EOL><DEDENT>


def register_block(self, block_type, block):

self._verify_block(block_type, block)<EOL>self._registry[block_type] = block<EOL>

Registers `block` to `block_type` in the registry.


if block_type not in self._registry:<EOL><INDENT>raise NotRegistered(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>block_type<EOL>)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>del self._registry[block_type]<EOL><DEDENT>

Unregisters the block associated with `block_type` from the registry.

If no block is registered to `block_type`, NotRegistered will raise.


try:<EOL><INDENT>return len(self.bot.guilds)<EOL><DEDENT>except AttributeError:<EOL><INDENT>return len(self.bot.servers)<EOL><DEDENT>

Gets the guild count from the Client/Bot object


return await self.http.get_weekend_status()<EOL>

This function is a coroutine.

        Gets weekend status from discordbots.org

        Returns
        =======

        weekend status: bool
            The boolean value of weekend status.


async def post_guild_count(<EOL>self,<EOL>shard_count: int = None,<EOL>shard_no: int = None<EOL>):

await self.http.post_guild_count(self.bot_id, self.guild_count(), shard_count, shard_no)<EOL>

This function is a coroutine.

        Posts the guild count to discordbots.org

        .. _0 based indexing : https://en.wikipedia.org/wiki/Zero-based_numbering

        Parameters
        ==========

        shard_count: int[Optional]
            The total number of shards.
        shard_no: int[Optional]
            The index of the current shard. DBL uses `0 based indexing`_ for shards.


async def get_guild_count(self, bot_id: int=None):

if bot_id is None:<EOL><INDENT>bot_id = self.bot_id<EOL><DEDENT>return await self.http.get_guild_count(bot_id)<EOL>

This function is a coroutine.

        Gets a guild count from discordbots.org

        Parameters
        ==========

        bot_id: int[Optional]
            The bot_id of the bot you want to lookup.
            Defaults to the Bot provided in Client init

        Returns
        =======

        stats: dict
            The guild count and shards of a bot.
            The date object is returned in a datetime.datetime object


async def get_upvote_info(self, bot_id):

return await self.http.get_upvote_info(bot_id)<EOL>

This function is a coroutine.

        Gets information about who upvoted a bot from discordbots.org

        .. note::

            This API endpoint is available to the owner of the bot only.

        Returns
        =======

        votes: dict
            Info about who upvoted your bot.


async def get_bot_info(self, bot_id: int = None):

if bot_id is None:<EOL><INDENT>bot_id = self.bot_id<EOL><DEDENT>return await self.http.get_bot_info(bot_id)<EOL>

This function is a coroutine.

        Gets information about a bot from discordbots.org

        Parameters
        ==========

        bot_id: int[Optional]
            The bot_id of the bot you want to lookup.

        Returns
        =======

        bot_info: dict
            Information on the bot you looked up.
            https://discordbots.org/api/docs#bots


async def get_bots(self, limit: int = <NUM_LIT:50>, offset: int = <NUM_LIT:0>):

return await self.http.get_bots(limit, offset)<EOL>

This function is a coroutine.

        Gets information about listed bots on discordbots.org

        Parameters
        ==========

        limit: int[Optional]
            The number of results you wish to lookup. Defaults to 50. Max 500.
        offset: int[Optional]
            The amount of bots to skip. Defaults to 0.

        Returns
        =======

        bots: dict
            Returns info on the bots on DBL.
            https://discordbots.org/api/docs#bots


async def get_user_info(self, user_id: int):

return await self.http.get_user_info(user_id)<EOL>

This function is a coroutine.

        Gets information about a user on discordbots.org

        Parameters
        ==========

        user_id: int
            The user_id of the user you wish to lookup.

        Returns
        =======

        user_data: dict
            Info about the user.
            https://discordbots.org/api/docs#users


async def generate_widget_large(<EOL>self,<EOL>bot_id: int = None,<EOL>top: str = '<STR_LIT>',<EOL>mid: str = '<STR_LIT>',<EOL>user: str = '<STR_LIT>',<EOL>cert: str = '<STR_LIT>',<EOL>data: str = '<STR_LIT>',<EOL>label: str = '<STR_LIT>',<EOL>highlight: str = '<STR_LIT>'<EOL>):

url = '<STR_LIT>'<EOL>if bot_id is None:<EOL><INDENT>bot_id = self.bot_id<EOL><DEDENT>url = url.format(bot_id, top, mid, user, cert, data, label, highlight)<EOL>return url<EOL>

This function is a coroutine.

        Generates a custom large widget. Do not add `#` to the color codes (e.g. #FF00FF become FF00FF).

        Parameters
        ==========

        bot_id: int
            The bot_id of the bot you wish to make a widget for.
        top: str
            The hex color code of the top bar.
        mid: str
            The hex color code of the main section.
        user: str
            The hex color code of the username text.
        cert: str
            The hex color code of the certified text (if applicable).
        data: str
            The hex color code of the statistics (numbers only e.g. 44) of the bot.
        label: str
            The hex color code of the description (text e.g. guild count) of the statistics.
        highlight: str
            The hex color code of the data boxes.

        Returns
        =======

        URL of the widget: str


async def get_widget_large(self, bot_id: int = None):

if bot_id is None:<EOL><INDENT>bot_id = self.bot_id<EOL><DEDENT>url = '<STR_LIT>'.format(bot_id)<EOL>return url<EOL>

This function is a coroutine.

        Generates the default large widget.

        Parameters
        ==========

        bot_id: int
            The bot_id of the bot you wish to make a widget for.

        Returns
        =======

        URL of the widget: str


async def generate_widget_small(<EOL>self,<EOL>bot_id: int = None,<EOL>avabg: str = '<STR_LIT>',<EOL>lcol: str = '<STR_LIT>',<EOL>rcol: str = '<STR_LIT>',<EOL>ltxt: str = '<STR_LIT>',<EOL>rtxt: str = '<STR_LIT>'<EOL>):

url = '<STR_LIT>'<EOL>if bot_id is None:<EOL><INDENT>bot_id = self.bot_id<EOL><DEDENT>url = url.format(bot_id, avabg, ltxt, rtxt, lcol, rcol)<EOL>return url<EOL>

This function is a coroutine.

        Generates a custom large widget. Do not add `#` to the color codes (e.g. #FF00FF become FF00FF).

        Parameters
        ==========

        bot_id: int
            The bot_id of the bot you wish to make a widget for.
        avabg: str
            The hex color code of the background of the avatar (if the avatar has transparency).
        lcol: str
            The hex color code of the left background color.
        rcol: str
            The hex color code of the right background color.
        ltxt: str
            The hex color code of the left text.
        rtxt: str
            The hex color code of the right text.

        Returns
        =======

        URL of the widget: str


async def get_widget_small(self, bot_id: int = None):

This function is a coroutine.

        Generates the default small widget.

        Parameters
        ==========

        bot_id: int
            The bot_id of the bot you wish to make a widget for.

        Returns
        =======

        URL of the widget: str


if self._is_closed:<EOL><INDENT>return<EOL><DEDENT>else:<EOL><INDENT>await self.http.close()<EOL>self._is_closed = True<EOL><DEDENT>

This function is a coroutine.

        Closes all connections.


text = await response.text()<EOL>if response.headers['<STR_LIT:Content-Type>'] == '<STR_LIT>':<EOL><INDENT>return json.loads(text)<EOL><DEDENT>return text<EOL>

Turns response into a properly formatted json or text object


duration = int(round(until - time.time()))<EOL>mins = duration / <NUM_LIT><EOL>fmt = '<STR_LIT>'<EOL>log.warn(fmt, duration, mins)<EOL>

Handles the message shown when we are ratelimited


async def request(self, method, url, **kwargs):

rate_limiter = RateLimiter(max_calls=<NUM_LIT>, period=<NUM_LIT>, callback=limited)<EOL>async with rate_limiter:  <EOL><INDENT>if not self.token:<EOL><INDENT>raise UnauthorizedDetected('<STR_LIT>')<EOL><DEDENT>headers = {<EOL>'<STR_LIT>': self.user_agent,<EOL>'<STR_LIT:Content-Type>': '<STR_LIT:application/json>'<EOL>}<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>kwargs['<STR_LIT:data>'] = to_json(kwargs.pop('<STR_LIT>'))<EOL><DEDENT>kwargs['<STR_LIT>'] = headers<EOL>headers['<STR_LIT>'] = self.token<EOL>for tries in range(<NUM_LIT:5>):<EOL><INDENT>async with self.session.request(method, url, **kwargs) as resp:<EOL><INDENT>log.debug('<STR_LIT>', method,<EOL>url, kwargs.get('<STR_LIT:data>'), resp.status)<EOL>data = await json_or_text(resp)<EOL>if <NUM_LIT> > resp.status >= <NUM_LIT:200>:<EOL><INDENT>return data<EOL><DEDENT>if resp.status == <NUM_LIT>:  <EOL><INDENT>fmt = '<STR_LIT>'<EOL>retry_after = json.loads(resp.headers.get('<STR_LIT>'))<EOL>mins = retry_after / <NUM_LIT><EOL>log.warning(fmt, retry_after, mins)<EOL>is_global = True  <EOL>if is_global:<EOL><INDENT>self._global_over.clear()<EOL><DEDENT>await asyncio.sleep(retry_after, loop=self.loop)<EOL>log.debug('<STR_LIT>')<EOL>if is_global:<EOL><INDENT>self._global_over.set()<EOL>log.debug('<STR_LIT>')<EOL><DEDENT>continue<EOL><DEDENT>if resp.status == <NUM_LIT>:<EOL><INDENT>raise HTTPException(resp, data)<EOL><DEDENT>elif resp.status == <NUM_LIT>:<EOL><INDENT>raise Unauthorized(resp, data)<EOL><DEDENT>elif resp.status == <NUM_LIT>:<EOL><INDENT>raise Forbidden(resp, data)<EOL><DEDENT>elif resp.status == <NUM_LIT>:<EOL><INDENT>raise NotFound(resp, data)<EOL><DEDENT>else:<EOL><INDENT>raise HTTPException(resp, data)<EOL><DEDENT><DEDENT><DEDENT>raise HTTPException(resp, data)<EOL><DEDENT>


return await self.request('<STR_LIT:GET>', '<STR_LIT>'.format(self.BASE))<EOL>


async def get_guild_count(self, bot_id):

return await self.request('<STR_LIT:GET>', '<STR_LIT>'.format(self.BASE, bot_id))<EOL>

Gets the guild count of the given Bot ID


resp = await self.request('<STR_LIT:GET>', '<STR_LIT>'.format(self.BASE, bot_id))<EOL>resp['<STR_LIT:date>'] = datetime.strptime(resp['<STR_LIT:date>'], '<STR_LIT>')<EOL>for k in resp:<EOL><INDENT>if resp[k] == '<STR_LIT>':<EOL><INDENT>resp[k] = None<EOL><DEDENT><DEDENT>return resp<EOL>

Gets the information of the given Bot ID


Gets the last 1000 votes of the given Bot ID


async def get_bots(self, limit, offset):

if limit > <NUM_LIT>:<EOL><INDENT>limit = <NUM_LIT:50><EOL><DEDENT>return await self.request('<STR_LIT:GET>', '<STR_LIT>'.format(self.BASE, limit, offset))<EOL>


return await self.request('<STR_LIT:GET>', '<STR_LIT>'.format(self.BASE, user_id))<EOL>


if isinstance(data, list):<EOL><INDENT>contents = data<EOL><DEDENT>else:<EOL><INDENT>contents = data.get('<STR_LIT>', [])<EOL>if isinstance(contents, six.string_types):<EOL><INDENT>contents = contents,<EOL><DEDENT><DEDENT>for content in contents:<EOL><INDENT>if isinstance(content, dict):<EOL><INDENT>content = self._create_bundle(content)<EOL><DEDENT>yield content<EOL><DEDENT>

Yield bundle contents from the given dict.

        Each item yielded will be either a string representing a file path
        or a bundle.


kwargs = {}<EOL>filters = None<EOL>if isinstance(data, dict):<EOL><INDENT>kwargs.update(<EOL>filters=data.get('<STR_LIT>', None),<EOL>output=data.get('<STR_LIT>', None),<EOL>debug=data.get('<STR_LIT>', None),<EOL>extra=data.get('<STR_LIT>', {}),<EOL>config=data.get('<STR_LIT>', {}),<EOL>depends=data.get('<STR_LIT>', None))<EOL><DEDENT>bundle = Bundle(*list(self._yield_bundle_contents(data)), **kwargs)<EOL>return self._auto_filter_bundle(bundle)<EOL>

Return a bundle initialised by the given dict.


def urls_for(self, asset_type, *args, **kwargs):

return self.urls_for_depends(asset_type, *args, **kwargs) +self.urls_for_self(asset_type, *args, **kwargs)<EOL>

Returns urls needed to include all assets of asset_type


def html_tags_for(self, asset_type, *args, **kwargs):

html = []<EOL>for ref in self.depends:<EOL><INDENT>html.append(self._ref(ref).html_tags_for(asset_type, *args, **kwargs))<EOL><DEDENT>if asset_type in self.typed_bundles:<EOL><INDENT>html.append(render_asset_html_tags(asset_type, self.urls_for_self(asset_type, *args, **kwargs)))<EOL><DEDENT>return "<STR_LIT:\n>".join(html)<EOL>


html = []<EOL>for asset_type in list_asset_types():<EOL><INDENT>html.append(self.html_tags_for(asset_type.name, *args, **kwargs))<EOL><DEDENT>return "<STR_LIT:\n>".join(html)<EOL>


for model in apps.get_app_config(appName).get_models():<EOL><INDENT>if model not in excludeModels:<EOL><INDENT>admin.site.register(model)<EOL><DEDENT><DEDENT>

Registers the models of the app with the given "appName" for the admin site


if e1.tag != e2.tag:<EOL><INDENT>return False<EOL><DEDENT>e1te = e1.text.strip() if e1.text is not None else '<STR_LIT>'<EOL>e2te = e2.text.strip() if e2.text is not None else '<STR_LIT>'<EOL>if e1te != e2te:<EOL><INDENT>return False<EOL><DEDENT>e1ta = e1.tail.strip() if e1.tail is not None else '<STR_LIT>'<EOL>e2ta = e2.tail.strip() if e2.tail is not None else '<STR_LIT>'<EOL>if e1ta != e2ta:<EOL><INDENT>return False<EOL><DEDENT>if e1.attrib != e2.attrib:<EOL><INDENT>return False<EOL><DEDENT>if len(e1) != len(e2):<EOL><INDENT>return False<EOL><DEDENT>return all(elements_equal(c1, c2) for c1, c2 in zip(e1, e2))<EOL>

Checks if two xml elements are equal.

    :param e1: First element to check.
    :type e1: etree._Element

    :param e2: Seconds element to check.
    :type e2: etree._Element

    Adapted from http://stackoverflow.com/a/24349916.

    :rtype: bool


if E is None:<EOL><INDENT>E = default_E<EOL><DEDENT>elif isinstance(E, str):<EOL><INDENT>E = ElementMaker(namespace=xml.openmath_ns,<EOL>nsmap={ E: xml.openmath_ns })<EOL><DEDENT>name = "<STR_LIT>"<EOL>attr = {}<EOL>children = []<EOL>if isinstance(obj, om.CDBaseAttribute) and obj.cdbase is not None:<EOL><INDENT>attr["<STR_LIT>"] = obj.cdbase<EOL><DEDENT>if isinstance(obj, om.CommonAttributes) and obj.id is not None:<EOL><INDENT>attr["<STR_LIT:id>"] = obj.id<EOL><DEDENT>if isinstance(obj, om.OMObject):<EOL><INDENT>children.append(encode_xml(obj.omel, E))<EOL>attr["<STR_LIT:version>"] = obj.version<EOL><DEDENT>elif isinstance(obj, om.OMReference):<EOL><INDENT>attr["<STR_LIT>"] = obj.href<EOL><DEDENT>elif isinstance(obj, om.OMInteger):<EOL><INDENT>children.append(str(obj.integer))<EOL><DEDENT>elif isinstance(obj, om.OMFloat):<EOL><INDENT>attr["<STR_LIT>"] = obj.double<EOL><DEDENT>elif isinstance(obj, om.OMString):<EOL><INDENT>if obj.string is not None:<EOL><INDENT>children.append(str(obj.string))<EOL><DEDENT><DEDENT>elif isinstance(obj, om.OMBytes):<EOL><INDENT>children.append(base64.b64encode(obj.bytes).decode('<STR_LIT:ascii>'))<EOL><DEDENT>elif isinstance(obj, om.OMSymbol):<EOL><INDENT>attr["<STR_LIT:name>"] = obj.name<EOL>attr["<STR_LIT>"] = obj.cd<EOL><DEDENT>elif isinstance(obj, om.OMVariable):<EOL><INDENT>attr["<STR_LIT:name>"] = obj.name<EOL><DEDENT>elif isinstance(obj, om.OMForeign):<EOL><INDENT>attr["<STR_LIT>"] = obj.encoding<EOL>children.append(str(obj.obj))<EOL><DEDENT>elif isinstance(obj, om.OMApplication):<EOL><INDENT>children = [encode_xml(obj.elem, E)]<EOL>children.extend(encode_xml(x, E) for x in obj.arguments)<EOL><DEDENT>elif isinstance(obj, om.OMAttribution):<EOL><INDENT>children = [encode_xml(obj.pairs, E), encode_xml(obj.obj, E)]<EOL><DEDENT>elif isinstance(obj, om.OMAttributionPairs):<EOL><INDENT>for (k, v) in obj.pairs:<EOL><INDENT>children.append(encode_xml(k, E))<EOL>children.append(encode_xml(v, E))<EOL><DEDENT><DEDENT>elif isinstance(obj, om.OMBinding):<EOL><INDENT>children = [<EOL>encode_xml(obj.binder, E),<EOL>encode_xml(obj.vars, E),<EOL>encode_xml(obj.obj, E)<EOL>]<EOL><DEDENT>elif isinstance(obj, om.OMBindVariables):<EOL><INDENT>children = [encode_xml(x, E) for x in obj.vars]<EOL><DEDENT>elif isinstance(obj, om.OMAttVar):<EOL><INDENT>children = [encode_xml(obj.pairs, E), encode_xml(obj.obj, E)]<EOL><DEDENT>elif isinstance(obj, om.OMError):<EOL><INDENT>children = [encode_xml(obj.name, E)]<EOL>children.extend(encode_xml(x, E) for x in obj.params)<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % obj.__class__.__name__)<EOL><DEDENT>attr = dict((k,str(v)) for k, v in attr.items() if v is not None)<EOL>return E(xml.object_to_tag(obj), *children, **attr)<EOL>

Encodes an OpenMath object as an XML node.

    :param obj: OpenMath object (or related item) to encode as XML.
    :type obj: OMAny

    :param ns: Namespace prefix to use for
        http://www.openmath.org/OpenMath", or None if default namespace.
    :type ns: str, None

    :return: The XML node representing the OpenMath data structure.
    :rtype: etree._Element


node = encode_xml(obj, nsprefix)<EOL>return etree.tostring(node)<EOL>

Encodes an OpenMath element into a string.

    :param obj: Object to encode as string.
    :type obj: OMAny

    :rtype: bytes


<EOL>if omobj.__class__ in self._omclass_to_py:<EOL><INDENT>return self._omclass_to_py[omobj.__class__](omobj)<EOL><DEDENT>elif isinstance(omobj, om.OMSymbol):<EOL><INDENT>return self._lookup_to_python(omobj.cdbase, omobj.cd, omobj.name)<EOL><DEDENT>elif isinstance(omobj, om.OMApplication):<EOL><INDENT>elem = self.to_python(omobj.elem)<EOL>arguments = [self.to_python(x) for x in omobj.arguments]<EOL>return elem(*arguments)<EOL><DEDENT>raise ValueError('<STR_LIT>' % omobj.__class__.__name__)<EOL>


for cl, conv in reversed(self._conv_to_om):<EOL><INDENT>if cl is None or isinstance(obj, cl):<EOL><INDENT>try:<EOL><INDENT>return conv(obj)<EOL><DEDENT>except CannotConvertError:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>return obj.__openmath__()<EOL><DEDENT>raise ValueError('<STR_LIT>' % obj)<EOL>


def register_to_openmath(self, py_class, converter):

if py_class is not None and not isclass(py_class):<EOL><INDENT>raise TypeError('<STR_LIT>' % py_class)<EOL><DEDENT>if not callable(converter) and not isinstance(converter, om.OMAny):<EOL><INDENT>raise TypeError('<STR_LIT>' % converter)<EOL><DEDENT>self._conv_to_om.append((py_class, converter))<EOL>

Register a conversion from Python to OpenMath

        :param py_class: A Python class the conversion is attached to, or None
        :type py_class: None, type

        :param converter: A conversion function or an OpenMath object
        :type converter: Callable, OMAny

        :rtype: None

        ``converter`` will used to convert any object of type ``py_class``,
        or any object if ``py_class`` is ``None``. If ``converter`` is an
        OpenMath object, it is returned immediately. If it is a callable, it
        is called with the Python object as paramter; in this case, it must
        either return an OpenMath object, or raise an exception.  The
        special exception ``CannotConvertError`` can be used to signify that
        ``converter`` does not know how to convert the current object, and that
        ``to_openmath`` shall continue with the other converters. Any other
        exception stops conversion immediately.

        Converters registered by this function are called in order from the
        most recent to the oldest.


def _deprecated_register_to_python(self, cd, name, converter=None):

if converter is None:<EOL><INDENT>if isclass(cd) and issubclass(cd, om.OMAny):<EOL><INDENT>self._conv_to_py[cd] = name<EOL><DEDENT>else:<EOL><INDENT>raise TypeError('<STR_LIT>' % cd)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if isinstance(cd, str) and isinstance(name, str):<EOL><INDENT>self._conv_sym_to_py[(cd, name)] = converter<EOL><DEDENT>else:<EOL><INDENT>raise TypeError('<STR_LIT>' % cd.__class__)<EOL><DEDENT><DEDENT>

Register a conversion from OpenMath to Python

        This function has two forms. A three-arguments one:

        :param cd: A content dictionary name
        :type cd: str

        :param name: A symbol name
        :type name: str

        :param converter: A conversion function, or a Python object
        :type: Callable, Any

        Any object of type ``openmath.OMSymbol``, with content
        dictionary equal to ``cd`` and name equal to ``name`` will be converted
        using ``converter``. Also, any object of type ``openmath.OMApplication``
        whose first child is an ``openmath.OMSymbol`` as above will be converted
        using ``converter``. If ``converter`` is a callable, it will be called with the
        OpenMath object as parameter; otherwise ``converter`` will be returned.

        In the two-argument form

        :param cd: A subclass of ``OMAny``
        :type cd: type

        :param name: A conversion function
        :type name: Callable

        Any object of type ``cd`` will be passed to ``name()``, and the
        result will be returned. This forms is mainly to override default
        conversions for basic OpenMath tags (OMInteger, OMString, etc.). It
        is discouraged to use it for ``OMSymbol`` and ``OMApplication``.


def _deprecated_register(self, py_class, to_om, om_cd, om_name, to_py=None):

self.register_to_python(om_cd, om_name, to_py)<EOL>self.register_to_openmath(py_class, to_om)<EOL>

This is a shorthand for:

   ``self.register_to_python(om_cd, om_name, to_py)``
   ``self.register_to_openmath(py_class, to_om)``


return [field for field in self._fields if getattr(self, field) is not None]<EOL>

Return the list of the names of the fields holding a non trivial value

The intention is that this would return the fields that don't
hold their default value.

    >>> OMSymbol('+', 'arith')._fields
    ['name', 'cd', 'id', 'cdbase']
    >>> OMSymbol('+', 'arith')._non_default_fields()
    ['name', 'cd']
    >>> OMSymbol('+', 'arith', cdbase='foo')._non_default_fields()
    ['name', 'cd', 'cdbase']


if self._print_multiline:<EOL><INDENT>newindent = indent+"<STR_LIT:U+0020>"<EOL>newline = "<STR_LIT:\n>"+newindent<EOL>separator = "<STR_LIT>"+newindent<EOL><DEDENT>else:<EOL><INDENT>newline = "<STR_LIT>"<EOL>separator = "<STR_LIT:U+002CU+0020>"<EOL>newindent = indent<EOL><DEDENT>return self.__class__.__name__+"<STR_LIT:(>" + newline +separator.join("<STR_LIT>".format(field,<EOL>self._print_field(getattr(self, field), indent=newindent, multiline=self._print_multiline))<EOL>for field in self._non_default_fields()) +"<STR_LIT:)>"<EOL>

Return a readable string representation

    >>> o = OMApplication(OMSymbol('+', 'arith'), [
    ...       OMApplication(OMSymbol('sin', 'transc1'), [OMVariable('x')]),
    ...       OMApplication(OMSymbol('cos', 'transc1'), [OMVariable('y')])])
    >>> print(o)
    OMApplication(
      elem=OMSymbol(name='+', cd='arith'),
      arguments=[
        OMApplication(
          elem=OMSymbol(name='sin', cd='transc1'),
          arguments=[OMVariable(name='x')]),
        OMApplication(
          elem=OMSymbol(name='cos', cd='transc1'),
          arguments=[OMVariable(name='y')])])

    >>> o = OMError(OMSymbol('DivisionByZero', 'aritherror'),
    ...        [
    ...            OMApplication(OMSymbol('divide', 'arith1'), [
    ...                OMVariable('x'),
    ...                OMInteger(0)
    ...            ])
    ...        ])
    >>> print(o)
    OMError(
      name=OMSymbol(name='DivisionByZero', cd='aritherror'),
      params=[OMApplication(
        elem=OMSymbol(name='divide', cd='arith1'),
        arguments=[
          OMVariable(name='x'),
          OMInteger(integer=0)])])


kwargs.update({"<STR_LIT>": self,  '<STR_LIT>': list(args)})<EOL>return OMApplication(**kwargs)<EOL>

Create an OpenMath Application Object using this object


if hasattr(x, "<STR_LIT>") and x._ishelper:<EOL><INDENT>return x._toOM()<EOL><DEDENT>elif isinstance(x, om.OMAny):<EOL><INDENT>return x<EOL><DEDENT>elif isinstance(x, six.integer_types):<EOL><INDENT>return om.OMInteger(x)<EOL><DEDENT>elif isinstance(x, float):<EOL><INDENT>return om.OMFloat(x)<EOL><DEDENT>elif isinstance(x, six.string_types):<EOL><INDENT>return om.OMString(x)<EOL><DEDENT>elif isinstance(x, WrappedHelper):<EOL><INDENT>return x.toOM()<EOL><DEDENT>elif inspect.isfunction(x):<EOL><INDENT>paramMap = inspect.signature(x).parameters<EOL>params = [v for k, v in six.iteritems(paramMap)]<EOL>posArgKinds = [inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD]<EOL>if not all([p.kind in posArgKinds for p in params]):<EOL><INDENT>raise CannotInterpretAsOpenMath("<STR_LIT>")<EOL><DEDENT>paramsOM = [om.OMVariable(name=p.name) for p in params]<EOL>bodyOM = interpretAsOpenMath(x(*paramsOM))<EOL>return OMBinding(om.OMSymbol(name="<STR_LIT>", cd="<STR_LIT>", cdbase="<STR_LIT>"), paramsOM, bodyOM)<EOL><DEDENT>else:<EOL><INDENT>raise CannotInterpretAsOpenMath("<STR_LIT>" + str(x))<EOL><DEDENT>

tries to convert a Python object into an OpenMath object
    this is not a replacement for using a Converter for exporting Python objects
    instead, it is used conveniently building OM objects in DSL embedded in Python
    inparticular, it converts Python functions into OMBinding objects using lambdaOM as the binder


<EOL>if hasattr(term, "<STR_LIT>") and term._ishelper or isinstance(term, om.OMAny):<EOL><INDENT>return interpretAsOpenMath(term)<EOL><DEDENT>if converter is not None:<EOL><INDENT>try:<EOL><INDENT>_converted = converter.to_openmath(term)<EOL><DEDENT>except Exception as e:<EOL><INDENT>_converted = None<EOL><DEDENT>if isinstance(_converted, om.OMAny):<EOL><INDENT>return _converted<EOL><DEDENT><DEDENT>return interpretAsOpenMath(term)<EOL>

Converts a term into OpenMath, using either a converter or the interpretAsOpenMath method


return '<STR_LIT>' % (self._cdbase, self._converter, self._cdhook, self._symbolhook)<EOL>

returns a unique representation of this object


returns a human-readable representation of this object


return CDBaseHelper('<STR_LIT>' % (self._cdbase, other), self._converter, self._cdhook, self._symbolhook)<EOL>

returns a new CDBaseHelper with other appended to the base url


return self.__div__(other)<EOL>


if self._cdhook is not None:<EOL><INDENT>return self._cdhook(self._cdbase, name, self._converter, self._symbolhook)<EOL><DEDENT>return CDHelper(self._cdbase, name, self._converter, self._symbolhook)<EOL>

returns a CDHelper object with the given name and this as the base


return self.__getattr__(name)<EOL>


return '<STR_LIT>' % (self._cdbase, self._cd, self._converter, self._hook)<EOL>


<EOL>if self._hook is not None:<EOL><INDENT>return self._hook(name, cd, cdbase, converter)<EOL><DEDENT>return OMSymbol(name=name, cd=self._cd, cdbase=self._cdbase, converter=self._converter)<EOL>

returns an OpenMath Symbol with self as the content dictonary and the given name


return self.__getattr__("<STR_LIT>")<EOL>

Turns this object into an OpenMath symbol


def decode_bytes(xml, validator=None, snippet=False):

return decode_stream(io.BytesIO(xml), validator, snippet)<EOL>

Decodes a stream into an OpenMath object.

    :param xml: XML to decode.
    :type xml: bytes

    :param validator: Validator to use.

    :param snippet: Is this an OpenMath snippet, or a full object?
    :type snippet: Bool

    :rtype: OMAny


def decode_stream(stream, validator=None, snippet=False):

<EOL>tree = etree.parse(stream)<EOL>if validator is not None:<EOL><INDENT>validator.assertValid(tree)<EOL><DEDENT>root = tree.getroot()<EOL>v = root.get("<STR_LIT:version>")<EOL>if not snippet and (not v or v != "<STR_LIT>"):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return decode_xml(root)<EOL>

Decodes a stream into an OpenMath object.

    :param stream: Stream to decode.
    :type stream: Any

    :param validator: Validator to use.

    :param snippet: Is this an OpenMath snippet, or a full object?
    :type snippet: Bool

    :rtype: OMAny


obj = xml.tag_to_object(elem)<EOL>attrs = {}<EOL>def a2d(*props):<EOL><INDENT>for p in props:<EOL><INDENT>attrs[p] = elem.get(p)<EOL><DEDENT><DEDENT>if issubclass(obj, om.CommonAttributes):<EOL><INDENT>a2d("<STR_LIT:id>")<EOL><DEDENT>if issubclass(obj, om.CDBaseAttribute):<EOL><INDENT>a2d("<STR_LIT>")<EOL><DEDENT>if issubclass(obj, om.OMObject):<EOL><INDENT>a2d("<STR_LIT:version>")<EOL>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:0>])<EOL><DEDENT>elif issubclass(obj, om.OMReference):<EOL><INDENT>a2d("<STR_LIT>")<EOL><DEDENT>elif issubclass(obj, om.OMInteger):<EOL><INDENT>attrs["<STR_LIT>"] = int(elem.text)<EOL><DEDENT>elif issubclass(obj, om.OMFloat):<EOL><INDENT>attrs["<STR_LIT>"] = float(elem.get('<STR_LIT>'))<EOL><DEDENT>elif issubclass(obj, om.OMString):<EOL><INDENT>attrs["<STR_LIT:string>"] = elem.text<EOL><DEDENT>elif issubclass(obj, om.OMBytes):<EOL><INDENT>try:<EOL><INDENT>attrs["<STR_LIT>"] = base64.b64decode(elem.text)<EOL><DEDENT>except TypeError:<EOL><INDENT>attrs["<STR_LIT>"] = base64.b64decode(bytes(elem.text, "<STR_LIT:ascii>"))<EOL><DEDENT><DEDENT>elif issubclass(obj, om.OMSymbol):<EOL><INDENT>a2d("<STR_LIT:name>", "<STR_LIT>")<EOL><DEDENT>elif issubclass(obj, om.OMVariable):<EOL><INDENT>a2d("<STR_LIT:name>")<EOL><DEDENT>elif issubclass(obj, om.OMForeign):<EOL><INDENT>attrs["<STR_LIT>"] = elem.text<EOL>a2d("<STR_LIT>")<EOL><DEDENT>elif issubclass(obj, om.OMApplication):<EOL><INDENT>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:0>])<EOL>attrs["<STR_LIT>"] = list(map(decode_xml, elem[<NUM_LIT:1>:]))<EOL><DEDENT>elif issubclass(obj, om.OMAttribution):<EOL><INDENT>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:0>])<EOL>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:1>])<EOL><DEDENT>elif issubclass(obj, om.OMAttributionPairs):<EOL><INDENT>if not _in_bind:<EOL><INDENT>attrs["<STR_LIT>"] = [(decode_xml(k), decode_xml(v)) for k, v in zip(elem[::<NUM_LIT:2>], elem[<NUM_LIT:1>::<NUM_LIT:2>])]<EOL><DEDENT>else:<EOL><INDENT>obj = om.OMAttVar<EOL>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:0>], True)<EOL>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:1>], True)<EOL><DEDENT><DEDENT>elif issubclass(obj, om.OMBinding):<EOL><INDENT>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:0>])<EOL>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:1>])<EOL>attrs["<STR_LIT>"] = decode_xml(elem[<NUM_LIT:2>])<EOL><DEDENT>elif issubclass(obj, om.OMBindVariables):<EOL><INDENT>attrs["<STR_LIT>"] = list(map(lambda x:decode_xml(x, True), elem[:]))<EOL><DEDENT>elif issubclass(obj, om.OMError):<EOL><INDENT>attrs["<STR_LIT:name>"] = decode_xml(elem[<NUM_LIT:0>])<EOL>attrs["<STR_LIT>"] = list(map(decode_xml, elem[<NUM_LIT:1>:]))<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % obj.__name__)<EOL><DEDENT>return obj(**attrs)<EOL>

Decodes an XML element into an OpenMath object.

    :param elem: Element to decode.
    :type elem: etree._Element

    :param _in_bind: Internal flag used to indicate if we should decode within
    an OMBind.
    :type _in_bind: bool

    :rtype: OMAny


relaxng_doc = etree.parse(resource_filename(__name__, '<STR_LIT>'))<EOL>return etree.RelaxNG(relaxng_doc)<EOL>

Load the normative validator provided by the OpenMath foundation.


<EOL>if module == '<STR_LIT>' and six.PY3:<EOL><INDENT>module = '<STR_LIT>'<EOL><DEDENT>module = importlib.import_module(module)<EOL>return getattr(module, name)<EOL>

Evaluate an OpenMath symbol describing a global Python object

EXAMPLES::

    >>> from openmath.convert_pickle import to_python
    >>> from openmath.convert_pickle  import load_python_global
    >>> load_python_global('math', 'sin')
    <built-in function sin>

    >>> from openmath import openmath as om
    >>> o = om.OMSymbol(cdbase="http://python.org/", cd='math', name='sin')
    >>> to_python(o)
    <built-in function sin>


return tuple(args)<EOL>

Construct a tuple from a sequence (not list!) of objects

EXAMPLES::

    >>> from openmath.convert_pickle import tuple_from_sequence
    >>> tuple_from_sequence(1, 2, 3)
    (1, 2, 3)
    >>> tuple([1, 2, 3])
    (1, 2, 3)


<EOL>setstate = getattr(inst, "<STR_LIT>", None)<EOL>if setstate:<EOL><INDENT>setstate(state)<EOL>return inst<EOL><DEDENT>slotstate = None<EOL>if isinstance(state, tuple) and len(state) == <NUM_LIT:2>:<EOL><INDENT>state, slotstate = state<EOL><DEDENT>if state:<EOL><INDENT>try:<EOL><INDENT>d = inst.__dict__<EOL>try:<EOL><INDENT>for k, v in six.iteritems(state):<EOL><INDENT>d[six.moves.intern(k)] = v<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>d.update(state)<EOL><DEDENT><DEDENT>except RuntimeError:<EOL><INDENT>for k, v in state.items():<EOL><INDENT>setattr(inst, k, v)<EOL><DEDENT><DEDENT><DEDENT>if slotstate:<EOL><INDENT>for k, v in slotstate.items():<EOL><INDENT>setattr(inst, k, v)<EOL><DEDENT><DEDENT>return inst<EOL>

Apply the setstate protocol to initialize `inst` from `state`.

INPUT:

- ``inst`` -- a raw instance of a class
- ``state`` -- the state to restore; typically a dictionary mapping attribute names to their values

EXAMPLES::

    >>> from openmath.convert_pickle import cls_build
    >>> class A(object): pass
    >>> inst = A.__new__(A)
    >>> state = {"foo": 1, "bar": 4}
    >>> inst2 = cls_build(inst,state)
    >>> inst is inst2
    True
    >>> inst.foo
    1
    >>> inst.bar
    4


return om.OMSymbol(cdbase=self._cdbase, cd=module, name=name)<EOL>

r"""
        Helper function to build an OMS object

        EXAMPLES::

            >>> from openmath.convert_pickle import PickleConverter
            >>> converter = PickleConverter()
            >>> o = converter.OMSymbol(module="foo.bar", name="baz"); o
            OMSymbol(name='baz', cd='foo.bar', id=None, cdbase='http://python.org/')


return self.OMSymbol('<STR_LIT>', name='<STR_LIT:none>')<EOL>

r"""
        Return an OM object for :obj:`None`

        EXAMPLES::

            >>> from openmath.convert_pickle  import PickleConverter
            >>> converter = PickleConverter()
            >>> o = converter.OMNone(); o
            OMSymbol(name='none', cd='Python', id=None, cdbase='http://python.org/')
            >>> converter.to_python(o)


if b:<EOL><INDENT>return self.OMSymbol("<STR_LIT>", "<STR_LIT:true>")<EOL><DEDENT>else:<EOL><INDENT>return self.OMSymbol("<STR_LIT>", "<STR_LIT:false>")<EOL><DEDENT>

r"""
        Return an OM object for booleans

        EXAMPLES::

            >>> from openmath.convert_pickle  import PickleConverter
            >>> converter = PickleConverter()
            >>> o = converter.OMBool(True); o
            OMSymbol(name='true', cd='Python', id=None, cdbase='http://python.org/')
            >>> converter.to_python(o)
            True
            >>> o = converter.OMBool(False); o
            OMSymbol(name='false', cd='Python', id=None, cdbase='http://python.org/')
            >>> converter.to_python(o)
            False


<EOL>return om.OMApplication(elem=om.OMSymbol(cdbase=self._cdbase, cd='<STR_LIT>', name='<STR_LIT:list>', ), arguments=l)<EOL>

Convert a list of OM objects into an OM object

EXAMPLES::

    >>> from openmath import openmath as om
    >>> from openmath.convert_pickle  import PickleConverter
    >>> converter = PickleConverter()
    >>> o = converter.OMList([om.OMInteger(2), om.OMInteger(2)]); o
    OMApplication(elem=OMSymbol(name='list', cd='Python', id=None, cdbase='http://python.org/'),
             arguments=[OMInteger(integer=2, id=None),
                        OMInteger(integer=2, id=None)],
             id=None, cdbase=None)
    >>> converter.to_python(o)
    [2, 2]


return om.OMApplication(elem=self.OMSymbol(module='<STR_LIT>', name='<STR_LIT>'),<EOL>arguments=l)<EOL>

Convert a tuple of OM objects into an OM object

EXAMPLES::

    >>> from openmath import openmath as om
    >>> from openmath.convert_pickle  import PickleConverter
    >>> converter = PickleConverter()
    >>> o = converter.OMTuple([om.OMInteger(2), om.OMInteger(3)]); o
    OMApplication(elem=OMSymbol(name='tuple', cd='Python', id=None, cdbase='http://python.org/'),
             arguments=[OMInteger(integer=2, id=None), OMInteger(integer=3, id=None)], id=None, cdbase=None)
    >>> converter.to_python(o)
    (2, 3)


return om.OMApplication(elem=self.OMSymbol(module='<STR_LIT>', name='<STR_LIT>'),<EOL>arguments=[self.OMTuple(item) for item in items])<EOL>

Convert a dictionary (or list of items thereof) of OM objects into an OM object

EXAMPLES::

    >>> from openmath import openmath as om
    >>> from openmath.convert_pickle  import PickleConverter
    >>> converter = PickleConverter()
    >>> a = om.OMInteger(1)
    >>> b = om.OMInteger(3)
    >>> o = converter.OMDict([(a,b), (b,b)]); print(o)
    OMApplication(
      elem=OMSymbol(name='dict', cd='Python', cdbase='http://python.org/'),
      arguments=[
        OMApplication(
          elem=OMSymbol(name='tuple', cd='Python', cdbase='http://python.org/'),
          arguments=[
            OMInteger(integer=1),
            OMInteger(integer=3)]),
        OMApplication(
          elem=OMSymbol(name='tuple', cd='Python', cdbase='http://python.org/'),
          arguments=[
            OMInteger(integer=3),
            OMInteger(integer=3)])])
    >>> converter.to_python(o)
    {1: 3, 3: 3}


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>self._init = True<EOL>self.version = <NUM_LIT:4><EOL>self.database = {<EOL>'<STR_LIT:version>': <NUM_LIT:4>,<EOL>'<STR_LIT:user>': {<EOL>'<STR_LIT:username>': '<STR_LIT>',<EOL>'<STR_LIT:password>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>},<EOL>'<STR_LIT>': [[]],<EOL>'<STR_LIT>': {},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>}<EOL>}<EOL>self.storage_path = Constant.storage_path<EOL>self.cookie_path = Constant.cookie_path<EOL>self.file = None<EOL>

Database stores every info.

version int
#if value in file is unequal to value defined in this class.
#An database update will be applied.
user dict:
    username str
    key str
collections list:
    collection_info(dict):
        collection_name str
        collection_type str
        collection_describe str
        collection_songs list:
            song_id(int)
songs dict:
    song_id(int) dict:
        song_id int
        artist str
        song_name str
        mp3_url str
        album_name str
        album_id str
        quality str
        lyric str
        tlyric str
player_info dict:
    player_list list:
        songs_id(int)
    playing_list list:
        songs_id(int)
    playing_mode int
    playing_offset int

:return:


def notify(msg, msg_type=<NUM_LIT:0>, t=None):

if platform.system() == '<STR_LIT>':<EOL><INDENT>command = notify_command_osx(msg, msg_type, t)<EOL><DEDENT>else:<EOL><INDENT>command = notify_command_linux(msg, t)<EOL><DEDENT>os.system(command.encode('<STR_LIT:utf-8>'))<EOL>

Show system notification with duration t (ms)


br_to_quality = {<NUM_LIT>: '<STR_LIT>', <NUM_LIT>: '<STR_LIT>'}<EOL>alters = NetEase().songs_detail_new_api(song_ids)<EOL>urls = [alter['<STR_LIT:url>'] for alter in alters]<EOL>return urls<EOL>


with open('<STR_LIT>') as input_file:<EOL><INDENT>for line in input_file:<EOL><INDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>return ast.parse(line).body[<NUM_LIT:0>].value.s<EOL><DEDENT><DEDENT><DEDENT>


return color + text + END<EOL>


with autoflake.open_with_encoding(<EOL>filename,<EOL>encoding=autoflake.detect_encoding(filename)) as f:<EOL><INDENT>return len(list(autoflake.check(f.read())))<EOL><DEDENT>


with autoflake.open_with_encoding(<EOL>filename,<EOL>encoding=autoflake.detect_encoding(filename)) as f:<EOL><INDENT>return f.readlines()<EOL><DEDENT>

Return contents of file as a list of lines.


import difflib<EOL>return '<STR_LIT>'.join(difflib.unified_diff(<EOL>readlines(before),<EOL>readlines(after),<EOL>before,<EOL>after))<EOL>


def run(filename, command, verbose=False, options=None):

if not options:<EOL><INDENT>options = []<EOL><DEDENT>import test_autoflake<EOL>with test_autoflake.temporary_directory() as temp_directory:<EOL><INDENT>temp_filename = os.path.join(temp_directory,<EOL>os.path.basename(filename))<EOL>import shutil<EOL>shutil.copyfile(filename, temp_filename)<EOL>if <NUM_LIT:0> != subprocess.call(shlex.split(command) +<EOL>['<STR_LIT>', temp_filename] +<EOL>options):<EOL><INDENT>sys.stderr.write('<STR_LIT>' + filename + '<STR_LIT:\n>')<EOL>return False<EOL><DEDENT>try:<EOL><INDENT>file_diff = diff(filename, temp_filename)<EOL>if verbose:<EOL><INDENT>sys.stderr.write(file_diff)<EOL><DEDENT>if check_syntax(filename):<EOL><INDENT>try:<EOL><INDENT>check_syntax(temp_filename, raise_error=True)<EOL><DEDENT>except (SyntaxError, TypeError,<EOL>UnicodeDecodeError, ValueError) as exception:<EOL><INDENT>sys.stderr.write('<STR_LIT>' + filename + '<STR_LIT:\n>' +<EOL>str(exception) + '<STR_LIT:\n>')<EOL>return False<EOL><DEDENT><DEDENT>before_count = pyflakes_count(filename)<EOL>after_count = pyflakes_count(temp_filename)<EOL>if verbose:<EOL><INDENT>print('<STR_LIT>', (before_count, after_count))<EOL><DEDENT>if file_diff and after_count > before_count:<EOL><INDENT>sys.stderr.write('<STR_LIT>' + filename + '<STR_LIT>')<EOL>return False<EOL><DEDENT><DEDENT>except IOError as exception:<EOL><INDENT>sys.stderr.write(str(exception) + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT>return True<EOL>

Run autoflake on file at filename.

    Return True on success.


with autoflake.open_with_encoding(<EOL>filename,<EOL>encoding=autoflake.detect_encoding(filename)) as input_file:<EOL><INDENT>try:<EOL><INDENT>compile(input_file.read(), '<STR_LIT>', '<STR_LIT>', dont_inherit=True)<EOL>return True<EOL><DEDENT>except (SyntaxError, TypeError, UnicodeDecodeError, ValueError):<EOL><INDENT>if raise_error:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>


Return processed arguments (options and positional arguments).


if args.files:<EOL><INDENT>dir_paths = args.files<EOL><DEDENT>else:<EOL><INDENT>dir_paths = [path for path in sys.path<EOL>if os.path.isdir(path)]<EOL><DEDENT>options = []<EOL>if args.expand_star_imports:<EOL><INDENT>options.append('<STR_LIT>')<EOL><DEDENT>if args.imports:<EOL><INDENT>options.append('<STR_LIT>' + args.imports)<EOL><DEDENT>if args.remove_all_unused_imports:<EOL><INDENT>options.append('<STR_LIT>')<EOL><DEDENT>if args.remove_duplicate_keys:<EOL><INDENT>options.append('<STR_LIT>')<EOL><DEDENT>if args.remove_unused_variables:<EOL><INDENT>options.append('<STR_LIT>')<EOL><DEDENT>filenames = dir_paths<EOL>completed_filenames = set()<EOL>while filenames:<EOL><INDENT>try:<EOL><INDENT>name = os.path.realpath(filenames.pop(<NUM_LIT:0>))<EOL>if not os.path.exists(name):<EOL><INDENT>continue<EOL><DEDENT>if name in completed_filenames:<EOL><INDENT>sys.stderr.write(<EOL>colored(<EOL>'<STR_LIT>' + name + '<STR_LIT:\n>',<EOL>YELLOW))<EOL>continue<EOL><DEDENT>else:<EOL><INDENT>completed_filenames.update(name)<EOL><DEDENT>if os.path.isdir(name):<EOL><INDENT>for root, directories, children in os.walk(unicode(name)):<EOL><INDENT>filenames += [os.path.join(root, f) for f in children<EOL>if f.endswith('<STR_LIT>') and<EOL>not f.startswith('<STR_LIT:.>')]<EOL>directories[:] = [d for d in directories<EOL>if not d.startswith('<STR_LIT:.>')]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>verbose_message = '<STR_LIT>' + name<EOL>sys.stderr.write(colored(verbose_message + '<STR_LIT:\n>', YELLOW))<EOL>if not run(os.path.join(name),<EOL>command=args.command,<EOL>verbose=args.verbose,<EOL>options=options):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>except (UnicodeDecodeError, UnicodeEncodeError) as exception:<EOL><INDENT>print(exception, file=sys.stderr)<EOL>continue<EOL><DEDENT><DEDENT>return True<EOL>

Run recursively run autoflake on directory of files.

    Return False if the fix results in broken syntax.


return <NUM_LIT:0> if check(process_args()) else <NUM_LIT:1><EOL>


for is_plat_spec in [True, False]:<EOL><INDENT>path = distutils.sysconfig.get_python_lib(standard_lib=True,<EOL>plat_specific=is_plat_spec)<EOL>for name in os.listdir(path):<EOL><INDENT>yield name<EOL><DEDENT>try:<EOL><INDENT>for name in os.listdir(os.path.join(path, '<STR_LIT>')):<EOL><INDENT>yield name<EOL><DEDENT><DEDENT>except OSError:  <EOL><INDENT>pass<EOL><DEDENT><DEDENT>


for name in standard_paths():<EOL><INDENT>if name.startswith('<STR_LIT:_>') or '<STR_LIT:->' in name:<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT:.>' in name and name.rsplit('<STR_LIT:.>')[-<NUM_LIT:1>] not in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>continue<EOL><DEDENT>yield name.split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL><DEDENT>


for message in messages:<EOL><INDENT>if isinstance(message, pyflakes.messages.UnusedImport):<EOL><INDENT>yield message.lineno<EOL><DEDENT><DEDENT>


pattern = r'<STR_LIT>'<EOL>for message in messages:<EOL><INDENT>if isinstance(message, pyflakes.messages.UnusedImport):<EOL><INDENT>module_name = re.search(pattern, str(message))<EOL>module_name = module_name.group()[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>if module_name:<EOL><INDENT>yield (message.lineno, module_name)<EOL><DEDENT><DEDENT><DEDENT>

Yield line number and module name of unused imports.


for message in messages:<EOL><INDENT>if isinstance(message, pyflakes.messages.ImportStarUsed):<EOL><INDENT>yield message.lineno<EOL><DEDENT><DEDENT>


for message in messages:<EOL><INDENT>if isinstance(message, pyflakes.messages.ImportStarUsage):<EOL><INDENT>undefined_name = message.message_args[<NUM_LIT:0>]<EOL>module_name = message.message_args[<NUM_LIT:1>]<EOL>yield (message.lineno, undefined_name, module_name)<EOL><DEDENT><DEDENT>

Yield line number, undefined name, and its possible origin module.


for message in messages:<EOL><INDENT>if isinstance(message, pyflakes.messages.UnusedVariable):<EOL><INDENT>yield message.lineno<EOL><DEDENT><DEDENT>


messages = [<EOL>message for message in messages<EOL>if isinstance(message, pyflakes.messages.MultiValueRepeatedKeyLiteral)]<EOL>if messages:<EOL><INDENT>key_to_messages = create_key_to_messages_dict(messages)<EOL>lines = source.split('<STR_LIT:\n>')<EOL>for (key, messages) in key_to_messages.items():<EOL><INDENT>good = True<EOL>for message in messages:<EOL><INDENT>line = lines[message.lineno - <NUM_LIT:1>]<EOL>key = message.message_args[<NUM_LIT:0>]<EOL>if not dict_entry_has_key(line, key):<EOL><INDENT>good = False<EOL><DEDENT><DEDENT>if good:<EOL><INDENT>for message in messages:<EOL><INDENT>yield message.lineno<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


dictionary = collections.defaultdict(lambda: [])<EOL>for message in messages:<EOL><INDENT>dictionary[message.message_args[<NUM_LIT:0>]].append(message)<EOL><DEDENT>return dictionary<EOL>

Return dict mapping the key to list of messages.


if sys.version_info[<NUM_LIT:0>] == <NUM_LIT:2> and isinstance(source, unicode):<EOL><INDENT>try:<EOL><INDENT>source = source.encode('<STR_LIT:utf-8>')<EOL><DEDENT>except UnicodeError:  <EOL><INDENT>return []<EOL><DEDENT><DEDENT>reporter = ListReporter()<EOL>try:<EOL><INDENT>pyflakes.api.check(source, filename='<STR_LIT>', reporter=reporter)<EOL><DEDENT>except (AttributeError, RecursionError, UnicodeDecodeError):<EOL><INDENT>pass<EOL><DEDENT>return reporter.messages<EOL>


assert '<STR_LIT:\\>' not in line<EOL>assert '<STR_LIT:(>' not in line<EOL>assert '<STR_LIT:)>' not in line<EOL>assert '<STR_LIT:;>' not in line<EOL>if line.lstrip().startswith(('<STR_LIT>', '<STR_LIT>')):<EOL><INDENT>word = line.split()[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>package = word.split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL>assert '<STR_LIT:U+0020>' not in package<EOL>return package<EOL>

Return package name in import statement.


for symbol in '<STR_LIT>':<EOL><INDENT>if symbol in line:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>if line.lstrip().startswith('<STR_LIT:>>'):<EOL><INDENT>return True<EOL><DEDENT>return multiline_statement(line, previous_line)<EOL>

Return True if import is spans multiples lines.


for symbol in '<STR_LIT>':<EOL><INDENT>if symbol in line:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>sio = io.StringIO(line)<EOL>try:<EOL><INDENT>list(tokenize.generate_tokens(sio.readline))<EOL>return previous_line.rstrip().endswith('<STR_LIT:\\>')<EOL><DEDENT>except (SyntaxError, tokenize.TokenError):<EOL><INDENT>return True<EOL><DEDENT>

Return True if this is part of a multiline statement.


(indentation, imports) = re.split(pattern=r'<STR_LIT>',<EOL>string=line, maxsplit=<NUM_LIT:1>)<EOL>base_module = re.search(pattern=r'<STR_LIT>',<EOL>string=indentation).group(<NUM_LIT:1>)<EOL>imports = re.split(pattern=r'<STR_LIT:U+002C>', string=imports.strip())<EOL>imports = [base_module + '<STR_LIT:.>' + x.strip() for x in imports]<EOL>filtered_imports = [x.replace(base_module + '<STR_LIT:.>', '<STR_LIT>')<EOL>for x in imports if x not in unused_module]<EOL>if not filtered_imports:<EOL><INDENT>return get_indentation(line) + '<STR_LIT>' + get_line_ending(line)<EOL><DEDENT>indentation += '<STR_LIT>'<EOL>return (<EOL>indentation +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(filtered_imports)) +<EOL>get_line_ending(line))<EOL>

Parse and filter ``from something import a, b, c``.

    Return line without unused import modules, or `pass` if all of the
    module in import is unused.


assert '<STR_LIT:\\>' not in line<EOL>assert '<STR_LIT:(>' not in line<EOL>assert '<STR_LIT:)>' not in line<EOL>assert '<STR_LIT:;>' not in line<EOL>assert '<STR_LIT:#>' not in line<EOL>assert not line.lstrip().startswith('<STR_LIT>')<EOL>newline = get_line_ending(line)<EOL>if not newline:<EOL><INDENT>return line<EOL><DEDENT>(indentation, imports) = re.split(pattern=r'<STR_LIT>',<EOL>string=line, maxsplit=<NUM_LIT:1>)<EOL>indentation += '<STR_LIT>'<EOL>assert newline<EOL>return '<STR_LIT>'.join([indentation + i.strip() + newline<EOL>for i in sorted(imports.split('<STR_LIT:U+002C>'))])<EOL>

Return line with imports on separate lines.


imports = SAFE_IMPORTS<EOL>if additional_imports:<EOL><INDENT>imports |= frozenset(additional_imports)<EOL><DEDENT>del additional_imports<EOL>messages = check(source)<EOL>if ignore_init_module_imports:<EOL><INDENT>marked_import_line_numbers = frozenset()<EOL><DEDENT>else:<EOL><INDENT>marked_import_line_numbers = frozenset(<EOL>unused_import_line_numbers(messages))<EOL><DEDENT>marked_unused_module = collections.defaultdict(lambda: [])<EOL>for line_number, module_name in unused_import_module_name(messages):<EOL><INDENT>marked_unused_module[line_number].append(module_name)<EOL><DEDENT>if expand_star_imports and not (<EOL>re.search(r'<STR_LIT>', source) or<EOL>re.search(r'<STR_LIT>', source)<EOL>):<EOL><INDENT>marked_star_import_line_numbers = frozenset(<EOL>star_import_used_line_numbers(messages))<EOL>if len(marked_star_import_line_numbers) > <NUM_LIT:1>:<EOL><INDENT>marked_star_import_line_numbers = frozenset()<EOL><DEDENT>else:<EOL><INDENT>undefined_names = []<EOL>for line_number, undefined_name, _in star_import_usage_undefined_name(messages):<EOL><INDENT>undefined_names.append(undefined_name)<EOL><DEDENT>if not undefined_names:<EOL><INDENT>marked_star_import_line_numbers = frozenset()<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>marked_star_import_line_numbers = frozenset()<EOL><DEDENT>if remove_unused_variables:<EOL><INDENT>marked_variable_line_numbers = frozenset(<EOL>unused_variable_line_numbers(messages))<EOL><DEDENT>else:<EOL><INDENT>marked_variable_line_numbers = frozenset()<EOL><DEDENT>if remove_duplicate_keys:<EOL><INDENT>marked_key_line_numbers = frozenset(<EOL>duplicate_key_line_numbers(messages, source))<EOL><DEDENT>else:<EOL><INDENT>marked_key_line_numbers = frozenset()<EOL><DEDENT>line_messages = get_messages_by_line(messages)<EOL>sio = io.StringIO(source)<EOL>previous_line = '<STR_LIT>'<EOL>for line_number, line in enumerate(sio.readlines(), start=<NUM_LIT:1>):<EOL><INDENT>if '<STR_LIT:#>' in line:<EOL><INDENT>yield line<EOL><DEDENT>elif line_number in marked_import_line_numbers:<EOL><INDENT>yield filter_unused_import(<EOL>line,<EOL>unused_module=marked_unused_module[line_number],<EOL>remove_all_unused_imports=remove_all_unused_imports,<EOL>imports=imports,<EOL>previous_line=previous_line)<EOL><DEDENT>elif line_number in marked_variable_line_numbers:<EOL><INDENT>yield filter_unused_variable(line)<EOL><DEDENT>elif line_number in marked_key_line_numbers:<EOL><INDENT>yield filter_duplicate_key(line, line_messages[line_number],<EOL>line_number, marked_key_line_numbers,<EOL>source)<EOL><DEDENT>elif line_number in marked_star_import_line_numbers:<EOL><INDENT>yield filter_star_import(line, undefined_names)<EOL><DEDENT>else:<EOL><INDENT>yield line<EOL><DEDENT>previous_line = line<EOL><DEDENT>


line_messages = {}<EOL>for message in messages:<EOL><INDENT>line_messages[message.lineno] = message<EOL><DEDENT>return line_messages<EOL>

Return dictionary that maps line number to message.


undefined_name = sorted(set(marked_star_import_undefined_name))<EOL>return re.sub(r'<STR_LIT>', '<STR_LIT:U+002CU+0020>'.join(undefined_name), line)<EOL>

Return line with the star import expanded.


def filter_unused_import(line, unused_module, remove_all_unused_imports,<EOL>imports, previous_line='<STR_LIT>'):

if multiline_import(line, previous_line):<EOL><INDENT>return line<EOL><DEDENT>is_from_import = line.lstrip().startswith('<STR_LIT>')<EOL>if '<STR_LIT:U+002C>' in line and not is_from_import:<EOL><INDENT>return break_up_import(line)<EOL><DEDENT>package = extract_package_name(line)<EOL>if not remove_all_unused_imports and package not in imports:<EOL><INDENT>return line<EOL><DEDENT>if '<STR_LIT:U+002C>' in line:<EOL><INDENT>assert is_from_import<EOL>return filter_from_import(line, unused_module)<EOL><DEDENT>else:<EOL><INDENT>return (get_indentation(line) +<EOL>'<STR_LIT>' +<EOL>get_line_ending(line))<EOL><DEDENT>

Return line if used, otherwise return None.


if re.match(EXCEPT_REGEX, line):<EOL><INDENT>return re.sub(r'<STR_LIT>', '<STR_LIT::>', line, count=<NUM_LIT:1>)<EOL><DEDENT>elif multiline_statement(line, previous_line):<EOL><INDENT>return line<EOL><DEDENT>elif line.count('<STR_LIT:=>') == <NUM_LIT:1>:<EOL><INDENT>split_line = line.split('<STR_LIT:=>')<EOL>assert len(split_line) == <NUM_LIT:2><EOL>value = split_line[<NUM_LIT:1>].lstrip()<EOL>if '<STR_LIT:U+002C>' in split_line[<NUM_LIT:0>]:<EOL><INDENT>return line<EOL><DEDENT>if is_literal_or_name(value):<EOL><INDENT>value = '<STR_LIT>' + get_line_ending(line)<EOL><DEDENT>return get_indentation(line) + value<EOL><DEDENT>else:<EOL><INDENT>return line<EOL><DEDENT>


def filter_duplicate_key(line, message, line_number, marked_line_numbers,<EOL>source, previous_line='<STR_LIT>'):

if marked_line_numbers and line_number == sorted(marked_line_numbers)[<NUM_LIT:0>]:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return line<EOL>

Return '' if first occurrence of the key otherwise return `line`.


if '<STR_LIT:#>' in line:<EOL><INDENT>return False<EOL><DEDENT>result = re.match(r'<STR_LIT>', line)<EOL>if not result:<EOL><INDENT>return False<EOL><DEDENT>try:<EOL><INDENT>candidate_key = ast.literal_eval(result.group(<NUM_LIT:1>))<EOL><DEDENT>except (SyntaxError, ValueError):<EOL><INDENT>return False<EOL><DEDENT>if multiline_statement(result.group(<NUM_LIT:2>)):<EOL><INDENT>return False<EOL><DEDENT>return candidate_key == key<EOL>

Return True if `line` is a dict entry that uses `key`.

    Return False for multiline cases where the line should not be removed by
    itself.


try:<EOL><INDENT>ast.literal_eval(value)<EOL>return True<EOL><DEDENT>except (SyntaxError, ValueError):<EOL><INDENT>pass<EOL><DEDENT>if value.strip() in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>return True<EOL><DEDENT>return re.match(r'<STR_LIT>', value)<EOL>

Return True if value is a literal or a name.


sio = io.StringIO(source)<EOL>previous_token_type = None<EOL>last_pass_row = None<EOL>last_pass_indentation = None<EOL>previous_line = '<STR_LIT>'<EOL>for token in tokenize.generate_tokens(sio.readline):<EOL><INDENT>token_type = token[<NUM_LIT:0>]<EOL>start_row = token[<NUM_LIT:2>][<NUM_LIT:0>]<EOL>line = token[<NUM_LIT:4>]<EOL>is_pass = (token_type == tokenize.NAME and line.strip() == '<STR_LIT>')<EOL>if (start_row - <NUM_LIT:1> == last_pass_row and<EOL>get_indentation(line) == last_pass_indentation and<EOL>token_type in ATOMS and<EOL>not is_pass):<EOL><INDENT>yield start_row - <NUM_LIT:1><EOL><DEDENT>if is_pass:<EOL><INDENT>last_pass_row = start_row<EOL>last_pass_indentation = get_indentation(line)<EOL><DEDENT>if (is_pass and<EOL>previous_token_type != tokenize.INDENT and<EOL>not previous_line.rstrip().endswith('<STR_LIT:\\>')):<EOL><INDENT>yield start_row<EOL><DEDENT>previous_token_type = token_type<EOL>previous_line = line<EOL><DEDENT>

Yield line numbers of unneeded "pass" statements.


try:<EOL><INDENT>marked_lines = frozenset(useless_pass_line_numbers(source))<EOL><DEDENT>except (SyntaxError, tokenize.TokenError):<EOL><INDENT>marked_lines = frozenset()<EOL><DEDENT>sio = io.StringIO(source)<EOL>for line_number, line in enumerate(sio.readlines(), start=<NUM_LIT:1>):<EOL><INDENT>if line_number not in marked_lines:<EOL><INDENT>yield line<EOL><DEDENT><DEDENT>

Yield code with useless "pass" lines removed.


if line.strip():<EOL><INDENT>non_whitespace_index = len(line) - len(line.lstrip())<EOL>return line[:non_whitespace_index]<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>


non_whitespace_index = len(line.rstrip()) - len(line)<EOL>if not non_whitespace_index:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return line[non_whitespace_index:]<EOL><DEDENT>


if not source:<EOL><INDENT>return source<EOL><DEDENT>if '<STR_LIT>' in source:<EOL><INDENT>remove_unused_variables = False<EOL><DEDENT>filtered_source = None<EOL>while True:<EOL><INDENT>filtered_source = '<STR_LIT>'.join(<EOL>filter_useless_pass('<STR_LIT>'.join(<EOL>filter_code(<EOL>source,<EOL>additional_imports=additional_imports,<EOL>expand_star_imports=expand_star_imports,<EOL>remove_all_unused_imports=remove_all_unused_imports,<EOL>remove_duplicate_keys=remove_duplicate_keys,<EOL>remove_unused_variables=remove_unused_variables,<EOL>ignore_init_module_imports=ignore_init_module_imports,<EOL>))))<EOL>if filtered_source == source:<EOL><INDENT>break<EOL><DEDENT>source = filtered_source<EOL><DEDENT>return filtered_source<EOL>

Return code with all filtering run on it.


def fix_file(filename, args, standard_out):

encoding = detect_encoding(filename)<EOL>with open_with_encoding(filename, encoding=encoding) as input_file:<EOL><INDENT>source = input_file.read()<EOL><DEDENT>original_source = source<EOL>isInitFile = os.path.basename(filename) == '<STR_LIT>'<EOL>if args.ignore_init_module_imports and isInitFile:<EOL><INDENT>ignore_init_module_imports = True<EOL><DEDENT>else:<EOL><INDENT>ignore_init_module_imports = False<EOL><DEDENT>filtered_source = fix_code(<EOL>source,<EOL>additional_imports=args.imports.split('<STR_LIT:U+002C>') if args.imports else None,<EOL>expand_star_imports=args.expand_star_imports,<EOL>remove_all_unused_imports=args.remove_all_unused_imports,<EOL>remove_duplicate_keys=args.remove_duplicate_keys,<EOL>remove_unused_variables=args.remove_unused_variables,<EOL>ignore_init_module_imports=ignore_init_module_imports,<EOL>)<EOL>if original_source != filtered_source:<EOL><INDENT>if args.check:<EOL><INDENT>standard_out.write('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if args.in_place:<EOL><INDENT>with open_with_encoding(filename, mode='<STR_LIT:w>',<EOL>encoding=encoding) as output_file:<EOL><INDENT>output_file.write(filtered_source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>diff = get_diff_text(<EOL>io.StringIO(original_source).readlines(),<EOL>io.StringIO(filtered_source).readlines(),<EOL>filename)<EOL>standard_out.write('<STR_LIT>'.join(diff))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if args.check:<EOL><INDENT>standard_out.write('<STR_LIT>')<EOL><DEDENT><DEDENT>


def open_with_encoding(filename, encoding, mode='<STR_LIT:r>',<EOL>limit_byte_check=-<NUM_LIT:1>):

if not encoding:<EOL><INDENT>encoding = detect_encoding(filename, limit_byte_check=limit_byte_check)<EOL><DEDENT>return io.open(filename, mode=mode, encoding=encoding,<EOL>newline='<STR_LIT>')<EOL>

Return opened file with a specific encoding.


try:<EOL><INDENT>with open(filename, '<STR_LIT:rb>') as input_file:<EOL><INDENT>encoding = _detect_encoding(input_file.readline)<EOL>with open_with_encoding(filename, encoding) as input_file:<EOL><INDENT>input_file.read(limit_byte_check)<EOL><DEDENT><DEDENT>return encoding<EOL><DEDENT>except (LookupError, SyntaxError, UnicodeDecodeError):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>


try:<EOL><INDENT>from lib2to3.pgen2 import tokenize as lib2to3_tokenize<EOL>encoding = lib2to3_tokenize.detect_encoding(readline)[<NUM_LIT:0>]<EOL>return encoding<EOL><DEDENT>except (LookupError, SyntaxError, UnicodeDecodeError):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>


newline = '<STR_LIT:\n>'<EOL>diff = difflib.unified_diff(<EOL>old, new,<EOL>'<STR_LIT>' + filename,<EOL>'<STR_LIT>' + filename,<EOL>lineterm=newline)<EOL>text = '<STR_LIT>'<EOL>for line in diff:<EOL><INDENT>text += line<EOL>if not line.endswith(newline):<EOL><INDENT>text += newline + r'<STR_LIT>' + newline<EOL><DEDENT><DEDENT>return text<EOL>

Return text of unified diff between old and new.


return set(text.strip() for text in string.split('<STR_LIT:U+002C>') if text.strip())<EOL>


if filename.endswith('<STR_LIT>'):<EOL><INDENT>return True<EOL><DEDENT>try:<EOL><INDENT>with open_with_encoding(<EOL>filename,<EOL>None,<EOL>limit_byte_check=MAX_PYTHON_FILE_DETECTION_BYTES) as f:<EOL><INDENT>text = f.read(MAX_PYTHON_FILE_DETECTION_BYTES)<EOL>if not text:<EOL><INDENT>return False<EOL><DEDENT>first_line = text.splitlines()[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>except (IOError, IndexError):<EOL><INDENT>return False<EOL><DEDENT>if not PYTHON_SHEBANG_REGEX.match(first_line):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>


base_name = os.path.basename(filename)<EOL>if base_name.startswith('<STR_LIT:.>'):<EOL><INDENT>return True<EOL><DEDENT>for pattern in exclude:<EOL><INDENT>if fnmatch.fnmatch(base_name, pattern):<EOL><INDENT>return True<EOL><DEDENT>if fnmatch.fnmatch(filename, pattern):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Return True if file matches exclude pattern.


if is_exclude_file(filename, exclude):<EOL><INDENT>return False<EOL><DEDENT>if not os.path.isdir(filename) and not is_python_file(filename):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Return True if file is okay for modifying/recursing.


def find_files(filenames, recursive, exclude):

while filenames:<EOL><INDENT>name = filenames.pop(<NUM_LIT:0>)<EOL>if recursive and os.path.isdir(name):<EOL><INDENT>for root, directories, children in os.walk(name):<EOL><INDENT>filenames += [os.path.join(root, f) for f in children<EOL>if match_file(os.path.join(root, f),<EOL>exclude)]<EOL>directories[:] = [d for d in directories<EOL>if match_file(os.path.join(root, d),<EOL>exclude)]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not is_exclude_file(name, exclude):<EOL><INDENT>yield name<EOL><DEDENT><DEDENT><DEDENT>


def _main(argv, standard_out, standard_error):

import argparse<EOL>parser = argparse.ArgumentParser(description=__doc__, prog='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT:-c>', '<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT:file>')<EOL>parser.add_argument('<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', action='<STR_LIT:version>',<EOL>version='<STR_LIT>' + __version__)<EOL>parser.add_argument('<STR_LIT>', nargs='<STR_LIT:+>', help='<STR_LIT>')<EOL>args = parser.parse_args(argv[<NUM_LIT:1>:])<EOL>if args.remove_all_unused_imports and args.imports:<EOL><INDENT>print('<STR_LIT>',<EOL>file=standard_error)<EOL>return <NUM_LIT:1><EOL><DEDENT>if args.exclude:<EOL><INDENT>args.exclude = _split_comma_separated(args.exclude)<EOL><DEDENT>else:<EOL><INDENT>args.exclude = set([])<EOL><DEDENT>filenames = list(set(args.files))<EOL>failure = False<EOL>for name in find_files(filenames, args.recursive, args.exclude):<EOL><INDENT>try:<EOL><INDENT>fix_file(name, args=args, standard_out=standard_out)<EOL><DEDENT>except IOError as exception:<EOL><INDENT>print(unicode(exception), file=standard_error)<EOL>failure = True<EOL><DEDENT><DEDENT>return <NUM_LIT:1> if failure else <NUM_LIT:0><EOL>

Return exit status.

    0 means no error.


try:<EOL><INDENT>signal.signal(signal.SIGPIPE, signal.SIG_DFL)<EOL><DEDENT>except AttributeError:  <EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>return _main(sys.argv,<EOL>standard_out=sys.stdout,<EOL>standard_error=sys.stderr)<EOL><DEDENT>except KeyboardInterrupt:  <EOL><INDENT>return <NUM_LIT:2><EOL><DEDENT>


ignore = StubFile()<EOL>pyflakes.reporter.Reporter.__init__(self, ignore, ignore)<EOL>self.messages = []<EOL>

Initialize.

        Ignore errors from Reporter.


self.messages.append(message)<EOL>


subprocess.check_call(['<STR_LIT>', '<STR_LIT>'.format(name=name)],<EOL>cwd=output_directory)<EOL>

Download package to output_directory.

    Raise CalledProcessError on failure.


if path.lower().endswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>tar = tarfile.open(path)<EOL>tar.extractall(path=output_directory)<EOL>tar.close()<EOL>return True<EOL><DEDENT>except (tarfile.ReadError, IOError):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>elif path.lower().endswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>archive = zipfile.ZipFile(path)<EOL>archive.extractall(path=output_directory)<EOL>archive.close()<EOL><DEDENT>except (zipfile.BadZipfile, IOError):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL><DEDENT>return False<EOL>


try:<EOL><INDENT>os.mkdir(TMP_DIR)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>args = test_fuzz.process_args()<EOL>if args.files:<EOL><INDENT>names = list(args.files)<EOL><DEDENT>else:<EOL><INDENT>names = None<EOL><DEDENT>checked_packages = []<EOL>skipped_packages = []<EOL>last_hours = <NUM_LIT:1><EOL>while True:<EOL><INDENT>if args.files:<EOL><INDENT>if not names:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while not names:<EOL><INDENT>names = [p for p in latest_packages(last_hours)<EOL>if p not in checked_packages and<EOL>p not in skipped_packages]<EOL>if not names:<EOL><INDENT>last_hours *= <NUM_LIT:2><EOL><DEDENT><DEDENT><DEDENT>package_name = names.pop(<NUM_LIT:0>)<EOL>print(package_name, file=sys.stderr)<EOL>package_tmp_dir = os.path.join(TMP_DIR, package_name)<EOL>try:<EOL><INDENT>os.mkdir(package_tmp_dir)<EOL><DEDENT>except OSError:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>skipped_packages.append(package_name)<EOL>continue<EOL><DEDENT>try:<EOL><INDENT>download_package(<EOL>package_name,<EOL>output_directory=package_tmp_dir)<EOL><DEDENT>except subprocess.CalledProcessError:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>continue<EOL><DEDENT>for download_name in os.listdir(package_tmp_dir):<EOL><INDENT>try:<EOL><INDENT>if not extract_package(<EOL>os.path.join(package_tmp_dir, download_name),<EOL>output_directory=package_tmp_dir):<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>continue<EOL><DEDENT><DEDENT>except UnicodeDecodeError:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>continue<EOL><DEDENT>args.files = [package_tmp_dir]<EOL>if test_fuzz.check(args):<EOL><INDENT>checked_packages.append(package_name)<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>if checked_packages:<EOL><INDENT>print('<STR_LIT>' + '<STR_LIT>'.join(checked_packages),<EOL>file=sys.stderr)<EOL><DEDENT>


@contextlib.contextmanager<EOL>def temporary_file(contents, directory='<STR_LIT:.>', suffix='<STR_LIT>', prefix='<STR_LIT>'):

f = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix,<EOL>delete=False, dir=directory)<EOL>try:<EOL><INDENT>f.write(contents.encode())<EOL>f.close()<EOL>yield f.name<EOL><DEDENT>finally:<EOL><INDENT>os.remove(f.name)<EOL><DEDENT>

Write contents to temporary file and yield it.


temp_directory = tempfile.mkdtemp(prefix=prefix, dir=directory)<EOL>try:<EOL><INDENT>yield temp_directory<EOL><DEDENT>finally:<EOL><INDENT>shutil.rmtree(temp_directory)<EOL><DEDENT>

Create temporary directory and yield its path.


if self.interval == <NUM_LIT:0.0>:<EOL><INDENT>return<EOL><DEDENT>self.short_interval -= self.short_unit<EOL>self.long_interval -= self.long_unit<EOL>self.short_interval = max(self.short_interval, Decimal(<NUM_LIT:0>))<EOL>self.long_interval = max(self.long_interval, Decimal(<NUM_LIT:0>))<EOL>self.update_interval()<EOL>

Update the timer to reflect a successfull call


self.short_interval += self.short_unit<EOL>self.long_interval += self.long_unit<EOL>self.short_interval = min(self.short_interval, self.max_short_timer)<EOL>self.long_interval = min(self.long_interval, self.max_long_timer)<EOL>self.update_interval()<EOL>

Update the timer to reflect a failed call


pass<EOL>

Called whenever a heartbeat has been received

This is useful to subclass and override to perform an action based on liveness (for
monitoring, etc.)

:param conn: the :class:`nsq.AsyncConn` over which the heartbeat was received


assert isinstance(params, dict)<EOL>encoded_params = []<EOL>for k, v in params.items():<EOL><INDENT>if v is None:<EOL><INDENT>continue<EOL><DEDENT>if isinstance(v, integer_types + (float,)):<EOL><INDENT>v = str(v)<EOL><DEDENT>if isinstance(v, (list, tuple)):<EOL><INDENT>v = [to_bytes(x) for x in v]<EOL><DEDENT>else:<EOL><INDENT>v = to_bytes(v)<EOL><DEDENT>encoded_params.append((k, v))<EOL><DEDENT>return dict(encoded_params)<EOL>

encode a dictionary of URL parameters (including iterables) as utf-8


for conn in self.conns.values():<EOL><INDENT>conn.close()<EOL><DEDENT>self.redist_periodic.stop()<EOL>if self.query_periodic is not None:<EOL><INDENT>self.query_periodic.stop()<EOL><DEDENT>

Closes all connections stops all periodic callbacks


assert callable(message_handler), '<STR_LIT>'<EOL>self.message_handler = message_handler<EOL>

Assigns the callback method to be executed for each message received

:param message_handler: a callable that takes a single argument


for conn in itervalues(self.conns):<EOL><INDENT>if conn.in_flight > <NUM_LIT:0> and conn.in_flight >= (conn.last_rdy * <NUM_LIT>):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Used to identify when buffered messages should be processed and responded to.

When max_in_flight > 1 and you're batching messages together to perform work
is isn't possible to just compare the len of your list of buffered messages against
your configured max_in_flight (because max_in_flight may not be evenly divisible
by the number of producers you're connected to, ie. you might never get that many
messages... it's a *max*).

Example::

    def message_handler(self, nsq_msg, reader):
        # buffer messages
        if reader.is_starved():
            # perform work

    reader = nsq.Reader(...)
    reader.set_message_handler(functools.partial(message_handler, reader=reader))
    nsq.run()


assert isinstance(host, string_types)<EOL>assert isinstance(port, int)<EOL>conn = AsyncConn(host, port, **self.conn_kwargs)<EOL>conn.on('<STR_LIT>', self._on_connection_identify)<EOL>conn.on('<STR_LIT>', self._on_connection_identify_response)<EOL>conn.on('<STR_LIT>', self._on_connection_auth)<EOL>conn.on('<STR_LIT>', self._on_connection_auth_response)<EOL>conn.on('<STR_LIT:error>', self._on_connection_error)<EOL>conn.on('<STR_LIT>', self._on_connection_close)<EOL>conn.on('<STR_LIT>', self._on_connection_ready)<EOL>conn.on('<STR_LIT:message>', self._on_message)<EOL>conn.on('<STR_LIT>', self._on_heartbeat)<EOL>conn.on('<STR_LIT>', functools.partial(self._on_backoff_resume, success=False))<EOL>conn.on('<STR_LIT>', functools.partial(self._on_backoff_resume, success=True))<EOL>conn.on('<STR_LIT>', functools.partial(self._on_backoff_resume, success=None))<EOL>if conn.id in self.conns:<EOL><INDENT>return<EOL><DEDENT>now = time.time()<EOL>last_connect_attempt = self.connection_attempts.get(conn.id)<EOL>if last_connect_attempt and last_connect_attempt > now - <NUM_LIT:10>:<EOL><INDENT>return<EOL><DEDENT>self.connection_attempts[conn.id] = now<EOL>logger.info('<STR_LIT>', conn.id, self.name)<EOL>conn.connect()<EOL>return conn<EOL>

Adds a connection to ``nsqd`` at the specified address.

:param host: the address to connect to
:param port: the port to connect to