def create_item(self, hash_key, start=<NUM_LIT:0>, extra_attrs=None):

table = self.get_table()<EOL>now = datetime.utcnow().replace(microsecond=<NUM_LIT:0>).isoformat()<EOL>attrs = {<EOL>'<STR_LIT>': now,<EOL>'<STR_LIT>': now,<EOL>'<STR_LIT:count>': start,<EOL>}<EOL>if extra_attrs:<EOL><INDENT>attrs.update(extra_attrs)<EOL><DEDENT>item = table.new_item(<EOL>hash_key=hash_key,<EOL>attrs=attrs,<EOL>)<EOL>return item<EOL>

Hook point for overriding how the CouterPool creates a DynamoDB item
for a given counter when an existing item can't be found.


def get_item(self, hash_key, start=<NUM_LIT:0>, extra_attrs=None):

table = self.get_table()<EOL>try:<EOL><INDENT>item = table.get_item(hash_key=hash_key)<EOL><DEDENT>except DynamoDBKeyNotFoundError:<EOL><INDENT>item = None<EOL><DEDENT>if item is None:<EOL><INDENT>item = self.create_item(<EOL>hash_key=hash_key,<EOL>start=start,<EOL>extra_attrs=extra_attrs,<EOL>)<EOL><DEDENT>return item<EOL>

Hook point for overriding how the CouterPool fetches a DynamoDB item
for a given counter.


def get_counter(self, name, start=<NUM_LIT:0>):

item = self.get_item(hash_key=name, start=start)<EOL>counter = Counter(dynamo_item=item, pool=self)<EOL>return counter<EOL>

Gets the DynamoDB item behind a counter and ties it to a Counter
instace.


if backend_name == '<STR_LIT>':<EOL><INDENT>lock_manager = _LockManagerRedis<EOL><DEDENT>elif backend_name == '<STR_LIT>':<EOL><INDENT>lock_manager = _LockManagerDB<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError<EOL><DEDENT>return lock_manager<EOL>

Select the proper LockManager based on the current backend used by Celery.

    :raise NotImplementedError: If Celery is using an unsupported backend.

    :param str backend_name: Class name of the current Celery backend. Usually value of
        current_app.extensions['celery'].celery.backend.__class__.__name__.

    :return: Class definition object (not instance). One of the _LockManager* classes.


def single_instance(func=None, lock_timeout=None, include_args=False):

if func is None:<EOL><INDENT>return partial(single_instance, lock_timeout=lock_timeout, include_args=include_args)<EOL><DEDENT>@wraps(func)<EOL>def wrapped(celery_self, *args, **kwargs):<EOL><INDENT>"""<STR_LIT>"""<EOL>timeout = (<EOL>lock_timeout or celery_self.soft_time_limit or celery_self.time_limit<EOL>or celery_self.app.conf.get('<STR_LIT>')<EOL>or celery_self.app.conf.get('<STR_LIT>')<EOL>or (<NUM_LIT> * <NUM_LIT:5>)<EOL>)<EOL>manager_class = _select_manager(celery_self.backend.__class__.__name__)<EOL>lock_manager = manager_class(celery_self, timeout, include_args, args, kwargs)<EOL>with lock_manager:<EOL><INDENT>ret_value = func(*args, **kwargs)<EOL><DEDENT>return ret_value<EOL><DEDENT>return wrapped<EOL>

Celery task decorator. Forces the task to have only one running instance at a time.

    Use with binded tasks (@celery.task(bind=True)).

    Modeled after:
    http://loose-bits.com/2010/10/distributed-task-locking-in-celery.html
    http://blogs.it.ox.ac.uk/inapickle/2012/01/05/python-decorators-with-optional-arguments/

    Written by @Robpol86.

    :raise OtherInstanceError: If another instance is already running.

    :param function func: The function to decorate, must be also decorated by @celery.task.
    :param int lock_timeout: Lock timeout in seconds plus five more seconds, in-case the task crashes and fails to
        release the lock. If not specified, the values of the task's soft/hard limits are used. If all else fails,
        timeout will be 5 minutes.
    :param bool include_args: Include the md5 checksum of the arguments passed to the task in the Redis key. This allows
        the same task to run with different arguments, only stopping a task from running if another instance of it is
        running with the same arguments.


def __init__(self, celery_self, timeout, include_args, args, kwargs):

self.celery_self = celery_self<EOL>self.timeout = timeout<EOL>self.include_args = include_args<EOL>self.args = args<EOL>self.kwargs = kwargs<EOL>self.log = getLogger('<STR_LIT>'.format(self.__class__.__name__, self.task_identifier))<EOL>

May raise NotImplementedError if the Celery backend is not supported.

        :param celery_self: From wrapped() within single_instance(). It is the `self` object specified in a binded
            Celery task definition (implicit first argument of the Celery task when @celery.task(bind=True) is used).
        :param int timeout: Lock's timeout value in seconds.
        :param bool include_args: If single instance should take arguments into account.
        :param iter args: The task instance's args.
        :param dict kwargs: The task instance's kwargs.


task_id = self.celery_self.name<EOL>if self.include_args:<EOL><INDENT>merged_args = str(self.args) + str([(k, self.kwargs[k]) for k in sorted(self.kwargs)])<EOL>task_id += '<STR_LIT>'.format(hashlib.md5(merged_args.encode('<STR_LIT:utf-8>')).hexdigest())<EOL><DEDENT>return task_id<EOL>

Return the unique identifier (string) of a task instance.


redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier)<EOL>return self.celery_self.backend.client.exists(redis_key)<EOL>

Return True if lock exists and has not timed out.


redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier)<EOL>self.celery_self.backend.client.delete(redis_key)<EOL>


date_done = (self.restore_group(self.task_identifier) or dict()).get('<STR_LIT>')<EOL>if not date_done:<EOL><INDENT>return False<EOL><DEDENT>difference = datetime.utcnow() - date_done<EOL>return difference < timedelta(seconds=self.timeout)<EOL>


self.delete_group(self.task_identifier)<EOL>


self.original_register_app = _state._register_app  <EOL>_state._register_app = lambda _: None  <EOL>super(Celery, self).__init__()<EOL>if app is not None:<EOL><INDENT>self.init_app(app)<EOL><DEDENT>

If app argument provided then initialize celery using application config values.

        If no app argument provided you should do initialization later with init_app method.

        :param app: Flask application instance.


_state._register_app = self.original_register_app  <EOL>if not hasattr(app, '<STR_LIT>'):<EOL><INDENT>app.extensions = dict()<EOL><DEDENT>if '<STR_LIT>' in app.extensions:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>app.extensions['<STR_LIT>'] = _CeleryState(self, app)<EOL>super(Celery, self).__init__(app.import_name, broker=app.config['<STR_LIT>'])<EOL>if '<STR_LIT>' in app.config:<EOL><INDENT>self._preconf['<STR_LIT>'] = app.config['<STR_LIT>']<EOL><DEDENT>self.conf.update(app.config)<EOL>task_base = self.Task<EOL>class ContextTask(task_base):<EOL><INDENT>def __call__(self, *_args, **_kwargs):<EOL><INDENT>with app.app_context():<EOL><INDENT>return task_base.__call__(self, *_args, **_kwargs)<EOL><DEDENT><DEDENT><DEDENT>setattr(ContextTask, '<STR_LIT>', True)<EOL>setattr(self, '<STR_LIT>', ContextTask)<EOL>

Actual method to read celery settings from app configuration and initialize the celery instance.

        :param app: Flask application instance.


pass<EOL>


Called when the Celery worker thread is ready to do work.

    This is to avoid race conditions since everything is in one python process.


celery_args = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT:-c>', '<STR_LIT:1>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>with app.app_context():<EOL><INDENT>celery.worker_main(celery_args)<EOL><DEDENT>


config = dict()<EOL>if os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>elif os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL>config['<STR_LIT>'] = config['<STR_LIT>']<EOL><DEDENT>elif os.environ.get('<STR_LIT>', '<STR_LIT>').startswith('<STR_LIT>'):<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>' + os.environ['<STR_LIT>'].split('<STR_LIT:U+002C>', <NUM_LIT:1>)[<NUM_LIT:1>]<EOL>config['<STR_LIT>'] = config['<STR_LIT>']<EOL><DEDENT>elif os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>elif os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>elif os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>elif os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>if os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>elif os.environ.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>config['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>file_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '<STR_LIT>')<EOL>config['<STR_LIT>'] = '<STR_LIT>' + file_path<EOL><DEDENT>config['<STR_LIT>'] = '<STR_LIT>' + config['<STR_LIT>']<EOL>config['<STR_LIT>'] = '<STR_LIT>' + config['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in config and '<STR_LIT>' not in config:<EOL><INDENT>config['<STR_LIT>'] = config['<STR_LIT>']<EOL><DEDENT>return config<EOL>

Generate a Flask config dict with settings for a specific broker based on an environment variable.

    To be merged into app.config.

    :return: Flask config to be fed into app.config.update().
    :rtype: dict


flask_app = Flask(__name__)<EOL>flask_app.config.update(config)<EOL>flask_app.config['<STR_LIT>'] = True<EOL>flask_app.config['<STR_LIT>'] = ['<STR_LIT>']<EOL>if '<STR_LIT>' in flask_app.config:<EOL><INDENT>db = SQLAlchemy(flask_app)<EOL>db.engine.execute('<STR_LIT>')<EOL><DEDENT>elif '<STR_LIT>' in flask_app.config:<EOL><INDENT>redis = Redis(flask_app)<EOL>redis.flushdb()<EOL><DEDENT>Celery(flask_app)<EOL>return flask_app<EOL>

Create the Flask app context and initializes any extensions such as Celery, Redis, SQLAlchemy, etc.

    :param dict config: Partial Flask config dict from generate_config().

    :return: The Flask app instance.


config = generate_config()<EOL>flask_app = generate_context(config=config)<EOL>celery_app = flask_app.extensions['<STR_LIT>'].celery<EOL>return flask_app, celery_app<EOL>

Call generate_context() and generate_config().

    :return: First item is the Flask app instance, second is the Celery app instance.
    :rtype: tuple


return x + y<EOL>


@celery.task(bind=True)<EOL>@single_instance(include_args=True, lock_timeout=<NUM_LIT:20>)<EOL>def mul(x, y):

return x * y<EOL>


return x - y<EOL>


@celery.task(bind=True, time_limit=<NUM_LIT>)<EOL>@single_instance<EOL>def add2(x, y):

return x + y<EOL>


@celery.task(bind=True, soft_time_limit=<NUM_LIT>)<EOL>@single_instance<EOL>def add3(x, y):

return x + y<EOL>


path = os.path.realpath(os.path.join(os.path.dirname(__file__), path))<EOL>handle = None<EOL>url_prefix = '<STR_LIT>'.format(name=NAME, version=VERSION)<EOL>try:<EOL><INDENT>handle = codecs.open(path, encoding='<STR_LIT:utf-8>')<EOL>return handle.read(<NUM_LIT>).replace('<STR_LIT>', '<STR_LIT>'.format(url_prefix))<EOL><DEDENT>except IOError:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>finally:<EOL><INDENT>getattr(handle, '<STR_LIT>', lambda: None)()<EOL><DEDENT>

Try to read README.rst or return empty string if failed.

    :param str path: Path to README file.

    :return: File contents.
    :rtype: str


project = __import__(IMPORT, fromlist=['<STR_LIT>'])<EOL>for expected, var in [('<STR_LIT>', '<STR_LIT>'), (LICENSE, '<STR_LIT>'), (VERSION, '<STR_LIT>')]:<EOL><INDENT>if getattr(project, var) != expected:<EOL><INDENT>raise SystemExit('<STR_LIT>'.format(var))<EOL><DEDENT><DEDENT>if not re.compile(r'<STR_LIT>' % VERSION, re.MULTILINE).search(readme()):<EOL><INDENT>raise SystemExit('<STR_LIT>')<EOL><DEDENT>if INSTALL_REQUIRES:<EOL><INDENT>contents = readme('<STR_LIT>')<EOL>section = re.compile(r'<STR_LIT>', re.DOTALL).findall(contents)<EOL>if not section:<EOL><INDENT>raise SystemExit('<STR_LIT>')<EOL><DEDENT>in_tox = re.findall(r'<STR_LIT>', section[<NUM_LIT:0>])<EOL>if INSTALL_REQUIRES != in_tox:<EOL><INDENT>raise SystemExit('<STR_LIT>')<EOL><DEDENT><DEDENT>


if not os.path.exists(file):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>config_parser = configparser.ConfigParser()<EOL>config_parser.read(file)<EOL>configuration = cls(file, config_parser)<EOL>if not configuration.check_config_sanity():<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return configuration<EOL><DEDENT><DEDENT>except configparser.Error:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Try loading given config file.

        :param str file: full path to the config file to load


file = os.path.join(Config.config_dir, Config.config_name)<EOL>return cls.from_file(file)<EOL>

Make a guess about the config file location an try loading it.


@classmethod<EOL><INDENT>def create_config(cls, cfgfile, nick, twtfile, twturl, disclose_identity, add_news):<DEDENT>

cfgfile_dir = os.path.dirname(cfgfile)<EOL>if not os.path.exists(cfgfile_dir):<EOL><INDENT>os.makedirs(cfgfile_dir)<EOL><DEDENT>cfg = configparser.ConfigParser()<EOL>cfg.add_section("<STR_LIT>")<EOL>cfg.set("<STR_LIT>", "<STR_LIT>", nick)<EOL>cfg.set("<STR_LIT>", "<STR_LIT>", twtfile)<EOL>cfg.set("<STR_LIT>", "<STR_LIT>", twturl)<EOL>cfg.set("<STR_LIT>", "<STR_LIT>", str(disclose_identity))<EOL>cfg.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>cfg.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>cfg.add_section("<STR_LIT>")<EOL>if add_news:<EOL><INDENT>cfg.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL><DEDENT>conf = cls(cfgfile, cfg)<EOL>conf.write_config()<EOL>return conf<EOL>

Create a new config file at the default location.

        :param str cfgfile: path to the config file
        :param str nick: nickname to use for own tweets
        :param str twtfile: path to the local twtxt file
        :param str twturl: URL to the remote twtxt file
        :param bool disclose_identity: if true the users id will be disclosed
        :param bool add_news: if true follow twtxt news feed


with open(self.config_file, "<STR_LIT:w>") as config_file:<EOL><INDENT>self.cfg.write(config_file)<EOL><DEDENT>

Writes `self.cfg` to `self.config_file`.


following = []<EOL>try:<EOL><INDENT>for (nick, url) in self.cfg.items("<STR_LIT>"):<EOL><INDENT>source = Source(nick, url)<EOL>following.append(source)<EOL><DEDENT><DEDENT>except configparser.NoSectionError as e:<EOL><INDENT>logger.debug(e)<EOL><DEDENT>return following<EOL>

A :class:`list` of all :class:`Source` objects.


try:<EOL><INDENT>return dict(self.cfg.items("<STR_LIT>"))<EOL><DEDENT>except configparser.NoSectionError as e:<EOL><INDENT>logger.debug(e)<EOL>return {}<EOL><DEDENT>


if not self.cfg.has_section("<STR_LIT>"):<EOL><INDENT>self.cfg.add_section("<STR_LIT>")<EOL><DEDENT>self.cfg.set("<STR_LIT>", source.nick, source.url)<EOL>self.write_config()<EOL>

Adds a new :class:`Source` to the config’s following section.


url = self.cfg.get("<STR_LIT>", nick, fallback=None)<EOL>return Source(nick, url) if url else None<EOL>

Returns the :class:`Source` of the given nick.

        :param str nick: nickname for which will be searched in the config


if not self.cfg.has_section("<STR_LIT>"):<EOL><INDENT>return False<EOL><DEDENT>ret_val = self.cfg.remove_option("<STR_LIT>", nick)<EOL>self.write_config()<EOL>return ret_val<EOL>

Removes a :class:`Source` form the config’s following section.

        :param str nick: nickname for which will be searched in the config


default_map = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": self.check_following,<EOL>"<STR_LIT>": self.timeout,<EOL>"<STR_LIT>": self.porcelain,<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": self.twtfile,<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": self.use_pager,<EOL>"<STR_LIT>": self.use_cache,<EOL>"<STR_LIT>": self.limit_timeline,<EOL>"<STR_LIT>": self.timeout,<EOL>"<STR_LIT>": self.sorting,<EOL>"<STR_LIT>": self.porcelain,<EOL>"<STR_LIT>": self.twtfile,<EOL>"<STR_LIT>": self.timeline_update_interval,<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": self.use_pager,<EOL>"<STR_LIT>": self.use_cache,<EOL>"<STR_LIT>": self.limit_timeline,<EOL>"<STR_LIT>": self.timeout,<EOL>"<STR_LIT>": self.sorting,<EOL>"<STR_LIT>": self.porcelain,<EOL>"<STR_LIT>": self.timeline_update_interval,<EOL>}<EOL>}<EOL>return default_map<EOL>

Maps config options to the default values used by click, returns :class:`dict`.


is_sane = True<EOL>properties = [property_name for property_name, obj<EOL>in self.__class__.__dict__.items()<EOL>if isinstance(obj, property)]<EOL>for property_name in properties:<EOL><INDENT>try:<EOL><INDENT>getattr(self, property_name)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>click.echo("<STR_LIT>".format(property_name, e))<EOL>is_sane = False<EOL><DEDENT><DEDENT>return is_sane<EOL>

Checks if the given values in the config file are sane.


def __init__(self, cache_file, cache, update_interval):

self.cache_file = cache_file<EOL>self.cache = cache<EOL>self.update_interval = update_interval<EOL>

Initializes new :class:`Cache` object.

        :param str cache_file: full path to the loaded cache file.
        :param ~shelve.Shelve cache: a Shelve object, with cache loaded.
        :param int update_interval: number of seconds the cache is considered to be
                                    up-to-date without calling any external resources.


@classmethod<EOL><INDENT>def from_file(cls, file, *args, **kwargs):<DEDENT>

try:<EOL><INDENT>cache = shelve.open(file)<EOL>return cls(file, cache, *args, **kwargs)<EOL><DEDENT>except OSError as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(file))<EOL>raise e<EOL><DEDENT>


@classmethod<EOL><INDENT>def discover(cls, *args, **kwargs):<DEDENT>

file = os.path.join(Cache.cache_dir, Cache.cache_name)<EOL>return cls.from_file(file, *args, **kwargs)<EOL>

Make a guess about the cache file location an try loading it.


try:<EOL><INDENT>return self.cache["<STR_LIT>"]<EOL><DEDENT>except KeyError:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>

Returns *NIX timestamp of last update of the cache.


if timestamp() - self.last_updated <= self.update_interval:<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Checks if the cache is considered to be up-to-date.


if not self.is_valid:<EOL><INDENT>self.cache["<STR_LIT>"] = timestamp()<EOL><DEDENT>

Mark cache as updated at current *NIX timestamp


try:<EOL><INDENT>return True if url in self.cache else False<EOL><DEDENT>except TypeError:<EOL><INDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>return self.cache[url]["<STR_LIT>"]<EOL><DEDENT>except KeyError:<EOL><INDENT>return None<EOL><DEDENT>

Returns saved 'Last-Modified' header, if available.


def add_tweets(self, url, last_modified, tweets):

try:<EOL><INDENT>self.cache[url] = {"<STR_LIT>": last_modified, "<STR_LIT>": tweets}<EOL>self.mark_updated()<EOL>return True<EOL><DEDENT>except TypeError:<EOL><INDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>tweets = self.cache[url]["<STR_LIT>"]<EOL>self.mark_updated()<EOL>return sorted(tweets, reverse=True)[:limit]<EOL><DEDENT>except KeyError:<EOL><INDENT>return []<EOL><DEDENT>


try:<EOL><INDENT>del self.cache[url]<EOL>self.mark_updated()<EOL>return True<EOL><DEDENT>except KeyError:<EOL><INDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>self.cache.close()<EOL>return True<EOL><DEDENT>except AttributeError:<EOL><INDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>self.cache.sync()<EOL>return True<EOL><DEDENT>except AttributeError:<EOL><INDENT>return False<EOL><DEDENT>


return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)<EOL>

Appends tzinfo and assumes UTC, if datetime object has no tzinfo already.


return make_aware(dateutil.parser.parse(string))<EOL>


def parse_tweets(raw_tweets, source, now=None):

if now is None:<EOL><INDENT>now = datetime.now(timezone.utc)<EOL><DEDENT>tweets = []<EOL>for line in raw_tweets:<EOL><INDENT>try:<EOL><INDENT>tweet = parse_tweet(line, source, now)<EOL><DEDENT>except (ValueError, OverflowError) as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(source.url, e))<EOL><DEDENT>else:<EOL><INDENT>tweets.append(tweet)<EOL><DEDENT><DEDENT>return tweets<EOL>

Parses a list of raw tweet lines from a twtxt file
and returns a list of :class:`Tweet` objects.

:param list raw_tweets: list of raw tweet lines
:param Source source: the source of the given tweets
:param Datetime now: the current datetime

:returns: a list of parsed tweets :class:`Tweet` objects
:rtype: list


def parse_tweet(raw_tweet, source, now=None):

if now is None:<EOL><INDENT>now = datetime.now(timezone.utc)<EOL><DEDENT>raw_created_at, text = raw_tweet.split("<STR_LIT:\t>", <NUM_LIT:1>)<EOL>created_at = parse_iso8601(raw_created_at)<EOL>if created_at > now:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return Tweet(click.unstyle(text.strip()), created_at, source)<EOL>

Parses a single raw tweet line from a twtxt file
and returns a :class:`Tweet` object.

:param str raw_tweet: a single raw tweet line
:param Source source: the source of the given tweet
:param Datetime now: the current datetime

:returns: the parsed tweet
:rtype: Tweet


@click.group()<EOL>@click.option("<STR_LIT>", "<STR_LIT:-c>",<EOL>type=click.Path(exists=True, file_okay=True, readable=True, writable=True, resolve_path=True),<EOL>help="<STR_LIT>")<EOL>@click.option("<STR_LIT>", "<STR_LIT>",<EOL>is_flag=True, default=False,<EOL>help="<STR_LIT>")<EOL>@click.version_option()<EOL>@click.pass_context<EOL>def cli(ctx, config, verbose):

init_logging(debug=verbose)<EOL>if ctx.invoked_subcommand == "<STR_LIT>":<EOL><INDENT>return  <EOL><DEDENT>try:<EOL><INDENT>if config:<EOL><INDENT>conf = Config.from_file(config)<EOL><DEDENT>else:<EOL><INDENT>conf = Config.discover()<EOL><DEDENT><DEDENT>except ValueError as e:<EOL><INDENT>if "<STR_LIT>" in str(e):<EOL><INDENT>click.echo("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>click.echo("<STR_LIT>")<EOL><DEDENT>sys.exit()<EOL><DEDENT>ctx.default_map = conf.build_default_map()<EOL>ctx.obj = {'<STR_LIT>': conf}<EOL>

Decentralised, minimalist microblogging service for hackers.


text = expand_mentions(text)<EOL>tweet = Tweet(text, created_at) if created_at else Tweet(text)<EOL>pre_tweet_hook = ctx.obj["<STR_LIT>"].pre_tweet_hook<EOL>if pre_tweet_hook:<EOL><INDENT>run_pre_tweet_hook(pre_tweet_hook, ctx.obj["<STR_LIT>"].options)<EOL><DEDENT>if not add_local_tweet(tweet, twtfile):<EOL><INDENT>click.echo("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>post_tweet_hook = ctx.obj["<STR_LIT>"].post_tweet_hook<EOL>if post_tweet_hook:<EOL><INDENT>run_post_tweet_hook(post_tweet_hook, ctx.obj["<STR_LIT>"].options)<EOL><DEDENT><DEDENT>


if source:<EOL><INDENT>source_obj = ctx.obj["<STR_LIT>"].get_source_by_nick(source)<EOL>if not source_obj:<EOL><INDENT>logger.debug("<STR_LIT>".format(source))<EOL>source_obj = Source(source, source)<EOL><DEDENT>sources = [source_obj]<EOL><DEDENT>else:<EOL><INDENT>sources = ctx.obj["<STR_LIT>"].following<EOL><DEDENT>tweets = []<EOL>if cache:<EOL><INDENT>try:<EOL><INDENT>with Cache.discover(update_interval=ctx.obj["<STR_LIT>"].timeline_update_interval) as cache:<EOL><INDENT>force_update = force_update or not cache.is_valid<EOL>if force_update:<EOL><INDENT>tweets = get_remote_tweets(sources, limit, timeout, cache)<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>cache.update_interval))<EOL>tweets = list(chain.from_iterable([cache.get_tweets(source.url) for source in sources]))<EOL><DEDENT><DEDENT><DEDENT>except OSError as e:<EOL><INDENT>logger.debug(e)<EOL>tweets = get_remote_tweets(sources, limit, timeout)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>tweets = get_remote_tweets(sources, limit, timeout)<EOL><DEDENT>if twtfile and not source:<EOL><INDENT>source = Source(ctx.obj["<STR_LIT>"].nick, ctx.obj["<STR_LIT>"].twturl, file=twtfile)<EOL>tweets.extend(get_local_tweets(source, limit))<EOL><DEDENT>if not tweets:<EOL><INDENT>return<EOL><DEDENT>tweets = sort_and_truncate_tweets(tweets, sorting, limit)<EOL>if pager:<EOL><INDENT>click.echo_via_pager(style_timeline(tweets, porcelain))<EOL><DEDENT>else:<EOL><INDENT>click.echo(style_timeline(tweets, porcelain))<EOL><DEDENT>


ctx.forward(timeline)<EOL>


sources = ctx.obj['<STR_LIT>'].following<EOL>if check:<EOL><INDENT>sources = get_remote_status(sources, timeout)<EOL>for (source, status) in sources:<EOL><INDENT>click.echo(style_source_with_status(source, status, porcelain))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sources = sorted(sources, key=lambda source: source.nick)<EOL>for source in sources:<EOL><INDENT>click.echo(style_source(source, porcelain))<EOL><DEDENT><DEDENT>

Return the list of sources you’re following.


source = Source(nick, url)<EOL>sources = ctx.obj['<STR_LIT>'].following<EOL>if not force:<EOL><INDENT>if source.nick in (source.nick for source in sources):<EOL><INDENT>click.confirm("<STR_LIT>".format(<EOL>click.style(source.nick, bold=True)), default=False, abort=True)<EOL><DEDENT>_, status = get_remote_status([source])[<NUM_LIT:0>]<EOL>if not status or status.status_code != <NUM_LIT:200>:<EOL><INDENT>click.confirm("<STR_LIT>".format(<EOL>click.style(source.nick, bold=True),<EOL>click.style(source.url, bold=True)), default=False, abort=True)<EOL><DEDENT><DEDENT>ctx.obj['<STR_LIT>'].add_source(source)<EOL>click.echo("<STR_LIT>".format(<EOL>click.style(source.nick, bold=True)))<EOL>


source = ctx.obj['<STR_LIT>'].get_source_by_nick(nick)<EOL>try:<EOL><INDENT>with Cache.discover() as cache:<EOL><INDENT>cache.remove_tweets(source.url)<EOL><DEDENT><DEDENT>except OSError as e:<EOL><INDENT>logger.debug(e)<EOL><DEDENT>ret_val = ctx.obj['<STR_LIT>'].remove_source_by_nick(nick)<EOL>if ret_val:<EOL><INDENT>click.echo("<STR_LIT>".format(<EOL>click.style(source.nick, bold=True)))<EOL><DEDENT>else:<EOL><INDENT>click.echo("<STR_LIT>".format(<EOL>click.style(nick, bold=True)))<EOL><DEDENT>

Remove an existing source from your followings.


width = click.get_terminal_size()[<NUM_LIT:0>]<EOL>width = width if width <= <NUM_LIT> else <NUM_LIT><EOL>click.secho("<STR_LIT>", fg="<STR_LIT>")<EOL>click.secho("<STR_LIT>", fg="<STR_LIT>")<EOL>click.echo()<EOL>help_text = "<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>click.echo(textwrap.fill(help_text, width))<EOL>click.echo()<EOL>nick = click.prompt("<STR_LIT>", default=os.environ.get("<STR_LIT>", "<STR_LIT>"))<EOL>def overwrite_check(path):<EOL><INDENT>if os.path.isfile(path):<EOL><INDENT>click.confirm("<STR_LIT>".format(path), abort=True)<EOL><DEDENT><DEDENT>cfgfile = click.prompt("<STR_LIT>",<EOL>os.path.join(Config.config_dir, Config.config_name),<EOL>type=click.Path(readable=True, writable=True, file_okay=True))<EOL>cfgfile = os.path.expanduser(cfgfile)<EOL>overwrite_check(cfgfile)<EOL>twtfile = click.prompt("<STR_LIT>",<EOL>os.path.expanduser("<STR_LIT>"),<EOL>type=click.Path(readable=True, writable=True, file_okay=True))<EOL>twtfile = os.path.expanduser(twtfile)<EOL>overwrite_check(twtfile)<EOL>twturl = click.prompt("<STR_LIT>",<EOL>default="<STR_LIT>")<EOL>disclose_identity = click.confirm("<STR_LIT>"<EOL>"<STR_LIT>", default=False)<EOL>click.echo()<EOL>add_news = click.confirm("<STR_LIT>", default=True)<EOL>conf = Config.create_config(cfgfile, nick, twtfile, twturl, disclose_identity, add_news)<EOL>twtfile_dir = os.path.dirname(twtfile)<EOL>if not os.path.exists(twtfile_dir):<EOL><INDENT>os.makedirs(twtfile_dir)<EOL><DEDENT>open(twtfile, "<STR_LIT:a>").close()<EOL>click.echo()<EOL>click.echo("<STR_LIT>".format(click.format_filename(conf.config_file)))<EOL>click.echo("<STR_LIT>".format(click.format_filename(twtfile)))<EOL>


conf = ctx.obj["<STR_LIT>"]<EOL>if not edit and not key:<EOL><INDENT>raise click.BadArgumentUsage("<STR_LIT>")<EOL><DEDENT>if edit:<EOL><INDENT>return click.edit(filename=conf.config_file)<EOL><DEDENT>if remove:<EOL><INDENT>try:<EOL><INDENT>conf.cfg.remove_option(key[<NUM_LIT:0>], key[<NUM_LIT:1>])<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug(e)<EOL><DEDENT>else:<EOL><INDENT>conf.write_config()<EOL><DEDENT>return<EOL><DEDENT>if not value:<EOL><INDENT>try:<EOL><INDENT>click.echo(conf.cfg.get(key[<NUM_LIT:0>], key[<NUM_LIT:1>]))<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug(e)<EOL><DEDENT>return<EOL><DEDENT>if not conf.cfg.has_section(key[<NUM_LIT:0>]):<EOL><INDENT>conf.cfg.add_section(key[<NUM_LIT:0>])<EOL><DEDENT>conf.cfg.set(key[<NUM_LIT:0>], key[<NUM_LIT:1>], value)<EOL>conf.write_config()<EOL>


if embed_names:<EOL><INDENT>mention_format = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>mention_format = "<STR_LIT>"<EOL><DEDENT>def handle_mention(match):<EOL><INDENT>source = get_source_by_name(match.group(<NUM_LIT:1>))<EOL>if source is None:<EOL><INDENT>return "<STR_LIT>".format(match.group(<NUM_LIT:1>))<EOL><DEDENT>return mention_format.format(<EOL>name=source.nick,<EOL>url=source.url)<EOL><DEDENT>return short_mention_re.sub(handle_mention, text)<EOL>

Searches the given text for mentions and expands them.

    For example:
    "@source.nick" will be expanded to "@<source.nick source.url>".


def handle_mention(match):<EOL><INDENT>name, url = match.groups()<EOL>return format_callback(name, url)<EOL><DEDENT>return mention_re.sub(handle_mention, text)<EOL>

Searches the given text for mentions generated by `expand_mention()` and returns a human-readable form.

    For example:
    "@<bob http://example.org/twtxt.txt>" will result in "@bob"

    If you follow a source: source.nick will be bold
    If you are the mentioned source: source.nick will be bold and coloured
    If nothing from the above is true: nick will be unstyled
    If nothing from the above is true and nick is not given: url will be used


def validate_config_key(ctx, param, value):

if not value:<EOL><INDENT>return value<EOL><DEDENT>try:<EOL><INDENT>section, item = value.split("<STR_LIT:.>", <NUM_LIT:1>)<EOL><DEDENT>except ValueError:<EOL><INDENT>raise click.BadArgumentUsage("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return section, item<EOL><DEDENT>

Validate a configuration key according to `section.item`.


now = datetime.now(timezone.utc)<EOL>tense = "<STR_LIT>" if self.created_at > now else "<STR_LIT>"<EOL>return "<STR_LIT>".format(humanize.naturaldelta(now - self.created_at), tense)<EOL>

Return human-readable relative time string.


return self.created_at.strftime("<STR_LIT>")<EOL>

Return human-readable absolute time string.


if (self.name is not None and self.version is not None<EOL>and self.version.startswith("<STR_LIT>")):<EOL><INDENT>return (self.__get_live_version() or self.__get_frozen_version()<EOL>or self.__fail_to_get_any_version())<EOL><DEDENT>else:<EOL><INDENT>return self.__base.get_version(self)<EOL><DEDENT>

Get distribution version.

This method is enhanced compared to original distutils implementation.
If the version string is set to a special value then instead of using
the actual value the real version is obtained by querying versiontools.

If versiontools package is not installed then the version is obtained
from the standard section of the ``PKG-INFO`` file. This file is
automatically created by any source distribution. This method is less
useful as it cannot take advantage of version control information that
is automatically loaded by versiontools. It has the advantage of not
requiring versiontools installation and that it does not depend on
``setup_requires`` feature of ``setuptools``.


try:<EOL><INDENT>import versiontools<EOL><DEDENT>except ImportError:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return str(versiontools.Version.from_expression(self.name))<EOL><DEDENT>

Get a live version string using versiontools


try:<EOL><INDENT>return self.__base("<STR_LIT>").version<EOL><DEDENT>except IOError:<EOL><INDENT>return None<EOL><DEDENT>

Get a fixed version string using an existing PKG-INFO file


raise SystemExit(<EOL>package requires versiontools for development or testing.<EOL>://versiontools.readthedocs.org/ for more information about<EOL>siontools is and why it is useful.<EOL>ll versiontools now please run:<EOL>p install versiontools<EOL>rsiontools works best when you have additional modules for<EOL>ing with your preferred version control system. Refer to<EOL>mentation for a full list of required modules.


return self<EOL>

Do nothing; this transformer is stateless.


return X[self.indices]<EOL>


raise NotImplementedError("<STR_LIT>")<EOL>


return X<EOL>

Does nothing, so that it works nicely in pipelines.


<EOL>n = X.shape[<NUM_LIT:0>]<EOL>if X.shape != (n, n):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>X = X + X.T<EOL>X /= <NUM_LIT:2><EOL>return X<EOL>

Symmetrizes X.

Parameters
----------
X : array, shape [n, n]
    The pairwise inputs to symmetrize.

Returns
-------
X : array, shape [n, n]
    The symmetrized pairwise outputs.


X = check_array(X)<EOL>if self.scale_by_median:<EOL><INDENT>self.median_ = np.median(X[np.triu_indices_from(X, k=<NUM_LIT:1>)],<EOL>overwrite_input=True)<EOL><DEDENT>elif hasattr(self, '<STR_LIT>'):<EOL><INDENT>del self.median_<EOL><DEDENT>return self<EOL>

If scale_by_median, find :attr:`median_`; otherwise, do nothing.

Parameters
----------
X : array
    The raw pairwise distances.


X = check_array(X)<EOL>X_rbf = np.empty_like(X) if self.copy else X<EOL>X_in = X<EOL>if not self.squared:<EOL><INDENT>np.power(X_in, <NUM_LIT:2>, out=X_rbf)<EOL>X_in = X_rbf<EOL><DEDENT>if self.scale_by_median:<EOL><INDENT>scale = self.median_ if self.squared else self.median_ ** <NUM_LIT:2><EOL>gamma = self.gamma * scale<EOL><DEDENT>else:<EOL><INDENT>gamma = self.gamma<EOL><DEDENT>np.multiply(X_in, -gamma, out=X_rbf)<EOL>np.exp(X_rbf, out=X_rbf)<EOL>return X_rbf<EOL>

Turns distances into RBF values.

Parameters
----------
X : array
    The raw pairwise distances.

Returns
-------
X_rbf : array of same shape as X
    The distances in X passed through the RBF kernel.


n = X.shape[<NUM_LIT:0>]<EOL>if X.shape != (n, n):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>memory = get_memory(self.memory)<EOL>vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['<STR_LIT>'])(<EOL>X, overwrite_a=not self.copy)<EOL>vals = vals.reshape(-<NUM_LIT:1>, <NUM_LIT:1>)<EOL>if self.min_eig == <NUM_LIT:0>:<EOL><INDENT>inner = vals > self.min_eig<EOL><DEDENT>else:<EOL><INDENT>with np.errstate(divide='<STR_LIT:ignore>'):<EOL><INDENT>inner = np.where(vals >= self.min_eig, <NUM_LIT:1>,<EOL>np.where(vals == <NUM_LIT:0>, <NUM_LIT:0>, self.min_eig / vals))<EOL><DEDENT><DEDENT>self.clip_ = np.dot(vecs, inner * vecs.T)<EOL>return self<EOL>

Learn the linear transformation to clipped eigenvalues.

Note that if min_eig isn't zero and any of the original eigenvalues
were exactly zero, this will leave those eigenvalues as zero.

Parameters
----------
X : array, shape [n, n]
    The *symmetric* input similarities. If X is asymmetric, it will be
    treated as if it were symmetric based on its lower-triangular part.


n = self.clip_.shape[<NUM_LIT:0>]<EOL>if X.ndim != <NUM_LIT:2> or X.shape[<NUM_LIT:1>] != n:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg.format(self.clip_.shape[<NUM_LIT:0>]))<EOL><DEDENT>return np.dot(X, self.clip_)<EOL>

Transforms X according to the linear transformation corresponding to
clipping the input eigenvalues.

Parameters
----------
X : array, shape [n_test, n]
    The test similarities to training points.

Returns
-------
Xt : array, shape [n_test, n]
    The transformed test similarites to training points.


n = X.shape[<NUM_LIT:0>]<EOL>if X.shape != (n, n):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>memory = get_memory(self.memory)<EOL>discard_X = not self.copy and self.negatives_likely<EOL>vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['<STR_LIT>'])(<EOL>X, overwrite_a=discard_X)<EOL>vals = vals[:, None]<EOL>self.clip_ = np.dot(vecs, (vals > self.min_eig) * vecs.T)<EOL>if discard_X or vals[<NUM_LIT:0>, <NUM_LIT:0>] < self.min_eig:<EOL><INDENT>del X<EOL>np.maximum(vals, self.min_eig, out=vals)<EOL>X = np.dot(vecs, vals * vecs.T)<EOL>del vals, vecs<EOL>X = Symmetrize(copy=False).fit_transform(X)<EOL><DEDENT>return X<EOL>

Clips the negative eigenvalues of X.

Parameters
----------
X : array, shape [n, n]
    The *symmetric* input similarities. If X is asymmetric, it will be
    treated as if it were symmetric based on its lower-triangular part.

Returns
-------
Xt : array, shape [n, n]
    The transformed training similarities; smallest eigenvalue will be
    at least `self.min_eig`.


n = X.shape[<NUM_LIT:0>]<EOL>if X.shape != (n, n):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>memory = get_memory(self.memory)<EOL>vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['<STR_LIT>'])(<EOL>X, overwrite_a=not self.copy)<EOL>vals = vals[:, None]<EOL>self.flip_ = np.dot(vecs, np.sign(vals) * vecs.T)<EOL>return self<EOL>

Learn the linear transformation to flipped eigenvalues.

Parameters
----------
X : array, shape [n, n]
    The *symmetric* input similarities. If X is asymmetric, it will be
    treated as if it were symmetric based on its lower-triangular part.


n = self.flip_.shape[<NUM_LIT:0>]<EOL>if X.ndim != <NUM_LIT:2> or X.shape[<NUM_LIT:1>] != n:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg.format(self.flip_.shape[<NUM_LIT:0>]))<EOL><DEDENT>return np.dot(X, self.flip_)<EOL>

Transforms X according to the linear transformation corresponding to
flipping the input eigenvalues.

Parameters
----------
X : array, shape [n_test, n]
    The test similarities to training points.

Returns
-------
Xt : array, shape [n_test, n]
    The transformed test similarites to training points.


n = X.shape[<NUM_LIT:0>]<EOL>if X.shape != (n, n):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>memory = get_memory(self.memory)<EOL>discard_X = not self.copy and self.negatives_likely<EOL>vals, vecs = memory.cache(scipy.linalg.eigh, ignore=['<STR_LIT>'])(<EOL>X, overwrite_a=discard_X)<EOL>vals = vals[:, None]<EOL>self.clip_ = np.dot(vecs, np.sign(vals) * vecs.T)<EOL>if discard_X or vals[<NUM_LIT:0>, <NUM_LIT:0>] < <NUM_LIT:0>:<EOL><INDENT>del X<EOL>np.abs(vals, out=vals)<EOL>X = np.dot(vecs, vals * vecs.T)<EOL>del vals, vecs<EOL>X = Symmetrize(copy=False).fit_transform(X)<EOL><DEDENT>return X<EOL>

Flips the negative eigenvalues of X.

Parameters
----------
X : array, shape [n, n]
    The *symmetric* input similarities. If X is asymmetric, it will be
    treated as if it were symmetric based on its lower-triangular part.

Returns
-------
Xt : array, shape [n, n]
    The transformed training similarities.


n = X.shape[<NUM_LIT:0>]<EOL>if X.shape != (n, n):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>self.train_ = X<EOL>memory = get_memory(self.memory)<EOL>lo, = memory.cache(scipy.linalg.eigvalsh)(X, eigvals=(<NUM_LIT:0>, <NUM_LIT:0>))<EOL>self.shift_ = max(self.min_eig - lo, <NUM_LIT:0>)<EOL>return self<EOL>

Learn the transformation to shifted eigenvalues. Only depends
on the input dimension.

Parameters
----------
X : array, shape [n, n]
    The *symmetric* input similarities.


n = self.train_.shape[<NUM_LIT:0>]<EOL>if X.ndim != <NUM_LIT:2> or X.shape[<NUM_LIT:1>] != n:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg.format(n))<EOL><DEDENT>if self.copy:<EOL><INDENT>X = X.copy()<EOL><DEDENT>if self.shift_ != <NUM_LIT:0> and X is self.train_ or (<EOL>X.shape == self.train_.shape and np.allclose(X, self.train_)):<EOL><INDENT>X[xrange(n), xrange(n)] += self.shift_<EOL><DEDENT>return X<EOL>

Transforms X according to the linear transformation corresponding to
shifting the input eigenvalues to all be at least ``self.min_eig``.

Parameters
----------
X : array, shape [n_test, n]
    The test similarities to training points.

Returns
-------
Xt : array, shape [n_test, n]
    The transformed test similarites to training points. Only different
    from X if X is the training data.


self.features_ = as_features(X, stack=True, bare=True)<EOL>return self<EOL>

Specify the data to which kernel values should be computed.

Parameters
----------
X : list of arrays or :class:`skl_groups.features.Features`
    The bags to compute "to".


X = as_features(X, stack=True, bare=True)<EOL>Y = self.features_<EOL>if X.dim != Y.dim:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>.format(X.dim, Y.dim))<EOL><DEDENT>pointwise = pairwise_kernels(X.stacked_features, Y.stacked_features,<EOL>metric=self.kernel,<EOL>filter_params=True,<EOL>**self._get_kernel_params())<EOL>K = np.empty((len(X), len(Y)))<EOL>for i in range(len(X)):<EOL><INDENT>for j in range(len(Y)):<EOL><INDENT>K[i, j] = pointwise[X._boundaries[i]:X._boundaries[i+<NUM_LIT:1>],<EOL>Y._boundaries[j]:Y._boundaries[j+<NUM_LIT:1>]].mean()<EOL><DEDENT><DEDENT>return K<EOL>

Compute kernels from X to :attr:`features_`.

Parameters
----------
X : list of arrays or :class:`skl_groups.features.Features`
    The bags to compute "from". Must have same dimension as
    :attr:`features_`.

Returns
-------
K : array of shape ``[len(X), len(features_)]``
    The kernel evaluations from X to :attr:`features_`.


X = as_features(X, stack=True)<EOL>self.transformer.fit(X.stacked_features, y, **params)<EOL>return self<EOL>

Fit the transformer on the stacked points.

Parameters
----------
X : :class:`Features` or list of arrays of shape ``[n_samples[i], n_features]``
    Training set. If a Features object, it will be stacked.

any other keyword argument :
    Passed on as keyword arguments to the transformer's ``fit()``.


X = as_features(X, stack=True)<EOL>X_new = self.transformer.transform(X.stacked_features, **params)<EOL>return self._gather_outputs(X, X_new)<EOL>

Transform the stacked points.

Parameters
----------
X : :class:`Features` or list of bag feature arrays
    New data to transform.

any other keyword argument :
    Passed on as keyword arguments to the transformer's ``transform()``.

Returns
-------
X_new : :class:`Features`
    Transformed features.


def fit_transform(self, X, y=None, **params):

X = as_features(X, stack=True)<EOL>X_new = self.transformer.fit_transform(X.stacked_features, y, **params)<EOL>return self._gather_outputs(X, X_new)<EOL>

Fit and transform the stacked points.

Parameters
----------
X : :class:`Features` or list of bag feature arrays
    Data to train on and transform.

any other keyword argument :
    Passed on as keyword arguments to the transformer's ``transform()``.

Returns
-------
X_new : :class:`Features`
    Transformed features.


def inverse_transform(self, X, **params):

X = as_features(X, stack=True)<EOL>Xo = self.transformer.inverse_transform(X.stacked_features, **params)<EOL>return self._gather_outputs(X, Xo)<EOL>

Transform data back to its original space, i.e., return an input
X_original whose transform would (maybe approximately) be X.

Parameters
----------
X : :class:`Features` or list of bag feature arrays
    Data to train on and transform.

any other keyword argument :
    Passed on as keyword arguments to the transformer's
    ``inverse_transform()``.

Returns
-------
X_original : :class:`Features`


X = check_array(X, copy=self.copy,<EOL>dtype=[np.float64, np.float32, np.float16, np.float128])<EOL>feature_range = self.feature_range<EOL>if feature_range[<NUM_LIT:0>] >= feature_range[<NUM_LIT:1>]:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>" % str(feature_range))<EOL><DEDENT>if self.fit_feature_range is not None:<EOL><INDENT>fit_feature_range = self.fit_feature_range<EOL>if fit_feature_range[<NUM_LIT:0>] >= fit_feature_range[<NUM_LIT:1>]:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>% str(feature_range))<EOL><DEDENT>if (fit_feature_range[<NUM_LIT:0>] < feature_range[<NUM_LIT:0>] or<EOL>fit_feature_range[<NUM_LIT:1>] > feature_range[<NUM_LIT:1>]):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>% (str(feature_range),<EOL>str(fit_feature_range)))<EOL><DEDENT>feature_range = fit_feature_range<EOL><DEDENT>data_min = np.min(X, axis=<NUM_LIT:0>)<EOL>data_range = np.max(X, axis=<NUM_LIT:0>) - data_min<EOL>data_range[data_range == <NUM_LIT:0.0>] = <NUM_LIT:1.0><EOL>self.scale_ = (feature_range[<NUM_LIT:1>] - feature_range[<NUM_LIT:0>]) / data_range<EOL>self.min_ = feature_range[<NUM_LIT:0>] - data_min * self.scale_<EOL>self.data_range = data_range<EOL>self.data_min = data_min<EOL>return self<EOL>

Compute the minimum and maximum to be used for later scaling.

        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to compute the per-feature minimum and maximum
            used for later scaling along the features axis.


X = check_array(X, copy=self.copy)<EOL>X *= self.scale_<EOL>X += self.min_<EOL>if self.truncate:<EOL><INDENT>np.maximum(self.feature_range[<NUM_LIT:0>], X, out=X)<EOL>np.minimum(self.feature_range[<NUM_LIT:1>], X, out=X)<EOL><DEDENT>return X<EOL>

Scaling features of X according to feature_range.

        Parameters
        ----------
        X : array-like with shape [n_samples, n_features]
            Input data that will be transformed.


X = check_array(X, copy=self.copy)<EOL>X -= self.min_<EOL>X /= self.scale_<EOL>return X<EOL>

Undo the scaling of X according to feature_range.

        Note that if truncate is true, any truncated points will not
        be restored exactly.

        Parameters
        ----------
        X : array-like with shape [n_samples, n_features]
            Input data that will be transformed.


def kl(Ks, dim, num_q, rhos, nus, clamp=True):

est = dim * np.mean(np.log(nus) - np.log(rhos), axis=<NUM_LIT:0>)<EOL>est += np.log(num_q / (rhos.shape[<NUM_LIT:0>] - <NUM_LIT:1>))<EOL>if clamp:<EOL><INDENT>np.maximum(est, <NUM_LIT:0>, out=est)<EOL><DEDENT>return est<EOL>

r'''
    Estimate the KL divergence between distributions:
        \int p(x) \log (p(x) / q(x))
    using the kNN-based estimator (5) of
        Qing Wang, Sanjeev R Kulkarni, and Sergio Verdu (2009).
        Divergence Estimation for Multidimensional Densities Via
        k-Nearest-Neighbor Distances.
        IEEE Transactions on Information Theory.
        http://www.ee.princeton.edu/~verdu/reprints/WanKulVer.May2009.pdf
    which is:
        d * 1/n \sum \log (nu_k(i) / rho_k(i)) + log(m / (n - 1))

    If clamp, enforces KL >= 0.

    Returns an array of shape (num_Ks,).


<EOL>logger.info("<STR_LIT>")<EOL>indices = [None] * len(X)<EOL>for i, bag in enumerate(plog(X, name="<STR_LIT>")):<EOL><INDENT>indices[i] = idx = FLANNIndex(**flann_args)<EOL>idx.build_index(bag)<EOL><DEDENT>return indices<EOL>


def _get_rhos(X, indices, Ks, max_K, save_all_Ks, min_dist):

logger.info("<STR_LIT>")<EOL>if max_K >= X.n_pts.min():<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(max_K, X.n_pts.min()))<EOL><DEDENT>which_Ks = slice(<NUM_LIT:1>, None) if save_all_Ks else Ks<EOL>indices = plog(indices, name="<STR_LIT>")<EOL>rhos = [None] * len(X)<EOL>for i, (idx, bag) in enumerate(zip(indices, X)):<EOL><INDENT>r = np.sqrt(idx.nn_index(bag, max_K + <NUM_LIT:1>)[<NUM_LIT:1>][:, which_Ks])<EOL>np.maximum(min_dist, r, out=r)<EOL>rhos[i] = r<EOL><DEDENT>return rhos<EOL>


return _get_linear(Ks, dim)(num_q, rhos, nus)<EOL>

r'''
    Estimates the linear inner product \int p q between two distributions,
    based on kNN distances.


def alpha_div(alphas, Ks, dim, num_q, rhos, nus):

return _get_alpha_div(alphas, Ks, dim)(num_q, rhos, nus)<EOL>

r'''
    Estimate the alpha divergence between distributions:
        \int p^\alpha q^(1-\alpha)
    based on kNN distances.

    Used in Renyi, Hellinger, Bhattacharyya, Tsallis divergences.

    Enforces that estimates are >= 0.

    Returns divergence estimates with shape (num_alphas, num_Ks).


def jensen_shannon_core(Ks, dim, num_q, rhos, nus):

ns = np.array([rhos.shape[<NUM_LIT:0>], num_q])<EOL>return _get_jensen_shannon_core(Ks, dim, ns)[<NUM_LIT:0>](num_q, rhos, nus)<EOL>

r'''
    Estimates
          1/2 mean_X( d * log radius of largest ball in X+Y around X_i
                                with no more than M/(n+m-1) weight
                                where X points have weight 1 / (2 n - 1)
                                  and Y points have weight n / (m (2 n - 1))
                      - digamma(# of neighbors in that ball))

    This is the core pairwise component of the estimator of Jensen-Shannon
    divergence based on the Hino-Murata weighted information estimator. See
    the docstring for jensen_shannon for an explanation.


def bhattacharyya(Ks, dim, required, clamp=True, to_self=False):

est = required<EOL>if clamp:<EOL><INDENT>est = np.minimum(est, <NUM_LIT:1>)  <EOL><DEDENT>return est<EOL>

r'''
    Estimate the Bhattacharyya coefficient between distributions, based on kNN
    distances:  \int \sqrt{p q}

    If clamp (the default), enforces 0 <= BC <= 1.

    Returns an array of shape (num_Ks,).


def hellinger(Ks, dim, required, clamp=True, to_self=False):

bc = required<EOL>est = <NUM_LIT:1> - bc<EOL>np.maximum(est, <NUM_LIT:0>, out=est)<EOL>if clamp:<EOL><INDENT>np.minimum(est, <NUM_LIT:1>, out=est)<EOL><DEDENT>np.sqrt(est, out=est)<EOL>return est<EOL>

r'''
    Estimate the Hellinger distance between distributions, based on kNN
    distances:  \sqrt{1 - \int \sqrt{p q}}

    Always enforces 0 <= H, to be able to sqrt; if clamp, also enforces
    H <= 1.

    Returns a vector: one element for each K.


def renyi(alphas, Ks, dim, required, min_val=np.spacing(<NUM_LIT:1>),<EOL>clamp=True, to_self=False):

alphas = np.reshape(alphas, (-<NUM_LIT:1>, <NUM_LIT:1>))<EOL>est = required<EOL>est = np.maximum(est, min_val)  <EOL>np.log(est, out=est)<EOL>est /= alphas - <NUM_LIT:1><EOL>if clamp:<EOL><INDENT>np.maximum(est, <NUM_LIT:0>, out=est)<EOL><DEDENT>return est<EOL>

r'''
    Estimate the Renyi-alpha divergence between distributions, based on kNN
    distances:  1/(\alpha-1) \log \int p^alpha q^(1-\alpha)

    If the inner integral is less than min_val (default ``np.spacing(1)``),
    uses the log of min_val instead.

    If clamp (the default), enforces that the estimates are nonnegative by
    replacing any negative estimates with 0.

    Returns an array of shape (num_alphas, num_Ks).


def tsallis(alphas, Ks, dim, required, clamp=True, to_self=False):

alphas = np.reshape(alphas, (-<NUM_LIT:1>, <NUM_LIT:1>))<EOL>alpha_est = required<EOL>est = alpha_est - <NUM_LIT:1><EOL>est /= alphas - <NUM_LIT:1><EOL>if clamp:<EOL><INDENT>np.maximum(est, <NUM_LIT:0>, out=est)<EOL><DEDENT>return est<EOL>

r'''
    Estimate the Tsallis-alpha divergence between distributions, based on kNN
    distances:  (\int p^alpha q^(1-\alpha) - 1) / (\alpha - 1)

    If clamp (the default), enforces the estimate is nonnegative.

    Returns an array of shape (num_alphas, num_Ks).


def l2(Ks, dim, X_rhos, Y_rhos, required, clamp=True, to_self=False):

n_X = len(X_rhos)<EOL>n_Y = len(Y_rhos)<EOL>linears = required<EOL>assert linears.shape == (<NUM_LIT:1>, Ks.size, n_X, n_Y, <NUM_LIT:2>)<EOL>X_quadratics = np.empty((Ks.size, n_X), dtype=np.float32)<EOL>for i, rho in enumerate(X_rhos):<EOL><INDENT>X_quadratics[:, i] = quadratic(Ks, dim, rho)<EOL><DEDENT>Y_quadratics = np.empty((Ks.size, n_Y), dtype=np.float32)<EOL>for j, rho in enumerate(Y_rhos):<EOL><INDENT>Y_quadratics[:, j] = quadratic(Ks, dim, rho)<EOL><DEDENT>est = -linears.sum(axis=<NUM_LIT:4>)<EOL>est += X_quadratics[None, :, :, None]<EOL>est += Y_quadratics[None, :, None, :]<EOL>np.maximum(est, <NUM_LIT:0>, out=est)<EOL>np.sqrt(est, out=est)<EOL>if to_self:<EOL><INDENT>est[:, :, xrange(n_X), xrange(n_Y)] = <NUM_LIT:0><EOL><DEDENT>return est[:, :, :, :, None]<EOL>

r'''
    Estimates the L2 distance between distributions, via
        \int (p - q)^2 = \int p^2 - \int p q - \int q p + \int q^2.

    \int pq and \int qp are estimated with the linear function (in both
    directions), while \int p^2 and \int q^2 are estimated via the quadratic
    function below.

    Always clamps negative estimates of l2^2 to 0, because otherwise the sqrt
    would break.


def quadratic(Ks, dim, rhos, required=None):

<EOL>N = rhos.shape[<NUM_LIT:0>]<EOL>Ks = np.asarray(Ks)<EOL>Bs = (Ks - <NUM_LIT:1>) / np.pi ** (dim / <NUM_LIT:2>) * gamma(dim / <NUM_LIT:2> + <NUM_LIT:1>)  <EOL>est = Bs / (N - <NUM_LIT:1>) * np.mean(rhos ** (-dim), axis=<NUM_LIT:0>)<EOL>return est<EOL>

r'''
    Estimates \int p^2 based on kNN distances.

    In here because it's used in the l2 distance, above.

    Returns array of shape (num_Ks,).


def jensen_shannon(Ks, dim, X_rhos, Y_rhos, required,<EOL>clamp=True, to_self=False):

X_ns = np.array([rho.shape[<NUM_LIT:0>] for rho in X_rhos])<EOL>Y_ns = np.array([rho.shape[<NUM_LIT:0>] for rho in Y_rhos])<EOL>n_X = X_ns.size<EOL>n_Y = Y_ns.size<EOL>cores = required<EOL>assert cores.shape == (<NUM_LIT:1>, Ks.size, n_X, n_Y, <NUM_LIT:2>)<EOL>X_bits = np.empty((Ks.size, n_X), dtype=np.float32)<EOL>for i, rho in enumerate(X_rhos):<EOL><INDENT>X_bits[:, i] = dim * np.mean(np.log(rho), axis=<NUM_LIT:0>)<EOL><DEDENT>X_bits += np.log(X_ns - <NUM_LIT:1>)[np.newaxis, :]<EOL>Y_bits = np.empty((Ks.size, n_Y), dtype=np.float32)<EOL>for j, rho in enumerate(Y_rhos):<EOL><INDENT>Y_bits[:, j] = dim * np.mean(np.log(rho), axis=<NUM_LIT:0>)<EOL><DEDENT>Y_bits += np.log(Y_ns - <NUM_LIT:1>)[np.newaxis, :]<EOL>est = cores.sum(axis=<NUM_LIT:4>)<EOL>est -= X_bits.reshape(<NUM_LIT:1>, Ks.size, n_X, <NUM_LIT:1>)<EOL>est -= Y_bits.reshape(<NUM_LIT:1>, Ks.size, <NUM_LIT:1>, n_Y)<EOL>est /= <NUM_LIT:2><EOL>est += np.log(-<NUM_LIT:1> + X_ns[None, None, :, None] + Y_ns[None, None, None, :])<EOL>est += psi(Ks)[None, :, None, None]<EOL>if to_self:<EOL><INDENT>est[:, :, xrange(n_X), xrange(n_Y)] = <NUM_LIT:0><EOL><DEDENT>if clamp:  <EOL><INDENT>np.maximum(<NUM_LIT:0>, est, out=est)<EOL>np.minimum(np.log(<NUM_LIT:2>), est, out=est)<EOL><DEDENT>return est[:, :, :, :, None]<EOL>

r'''
    Estimate the difference between the Shannon entropy of an equally-weighted
    mixture between X and Y and the mixture of the Shannon entropies:

        JS(X, Y) = H[ (X + Y) / 2 ] - (H[X] + H[Y]) / 2

    We use a special case of the Hino-Murata weighted information estimator with
    a fixed M = n \alpha, about equivalent to the K-nearest-neighbor approach
    used for the other estimators:

        Hideitsu Hino and Noboru Murata (2013).
        Information estimators for weighted observations. Neural Networks.
        http://linkinghub.elsevier.com/retrieve/pii/S0893608013001676

    The estimator for JS(X, Y) is:

        log volume of the unit ball - log M + log(n + m - 1) + digamma(M)
        + 1/2 mean_X( d * log radius of largest ball in X+Y around X_i
                                with no more than M/(n+m-1) weight
                                where X points have weight 1 / (2 n - 1)
                                  and Y points have weight n / (m (2 n - 1))
                      - digamma(# of neighbors in that ball) )
        + 1/2 mean_Y( d * log radius of largest ball in X+Y around Y_i
                                with no more than M/(n+m-1) weight
                                where X points have weight m / (n (2 m - 1))
                                  and Y points have weight 1 / (2 m - 1)
                      - digamma(# of neighbors in that ball) )

        - 1/2 (log volume of the unit ball - log M + log(n - 1) + digamma(M))
        - 1/2 mean_X( d * log radius of the largest ball in X around X_i
                                with no more than M/(n-1) weight
                                where X points have weight 1 / (n - 1))
                      - digamma(# of neighbors in that ball) )

        - 1/2 (log volume of the unit ball - log M + log(m - 1) + digamma(M))
        - 1/2 mean_Y( d * log radius of the largest ball in Y around Y_i
                                with no more than M/(n-1) weight
                                where X points have weight 1 / (m - 1))
                      - digamma(# of neighbors in that ball) )

        =

        log(n + m - 1) + digamma(M)
        + 1/2 mean_X( d * log radius of largest ball in X+Y around X_i
                                with no more than M/(n+m-1) weight
                                where X points have weight 1 / (2 n - 1)
                                  and Y points have weight n / (m (2 n - 1))
                      - digamma(# of neighbors in that ball) )
        + 1/2 mean_Y( d * log radius of largest ball in X+Y around Y_i
                                with no more than M/(n+m-1) weight
                                where X points have weight m / (n (2 m - 1))
                                  and Y points have weight 1 / (2 m - 1)
                      - digamma(# of neighbors in that ball) )
        - 1/2 [log(n-1) + mean_X( d * log rho_M(X_i) )]
        - 1/2 [log(m-1) + mean_Y( d * log rho_M(Y_i) )]


order = []<EOL>available = set()<EOL>def _move_available():<EOL><INDENT>to_delete = []<EOL>for n, parents in iteritems(deps):<EOL><INDENT>if not parents:<EOL><INDENT>available.add(n)<EOL>to_delete.append(n)<EOL><DEDENT><DEDENT>for n in to_delete:<EOL><INDENT>del deps[n]<EOL><DEDENT><DEDENT>_move_available()<EOL>while available:<EOL><INDENT>n = available.pop()<EOL>order.append(n)<EOL>for parents in itervalues(deps):<EOL><INDENT>parents.discard(n)<EOL><DEDENT>_move_available()<EOL><DEDENT>if available:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return order<EOL>

Topologically sort a DAG, represented by a dict of child => set of parents.
The dependency dict is destroyed during operation.

Uses the Kahn algorithm: http://en.wikipedia.org/wiki/Topological_sorting
Not a particularly good implementation, but we're just running it on tiny
graphs.


funcs = {}<EOL>metas = {}<EOL>meta_deps = defaultdict(set)<EOL>def add_func(func, alpha=None, pos=None):<EOL><INDENT>needs_alpha = getattr(func, '<STR_LIT>', False)<EOL>is_meta = hasattr(func, '<STR_LIT>')<EOL>d = metas if is_meta else funcs<EOL>if func not in d:<EOL><INDENT>if needs_alpha:<EOL><INDENT>args = {'<STR_LIT>': [alpha], '<STR_LIT>': [pos]}<EOL><DEDENT>else:<EOL><INDENT>args = {'<STR_LIT>': None, '<STR_LIT>': [pos]}<EOL><DEDENT>if not is_meta:<EOL><INDENT>d[func] = _FuncInfo(**args)<EOL><DEDENT>else:<EOL><INDENT>d[func] = _MetaFuncInfo(deps=[], **args)<EOL>for req in func.needs_results:<EOL><INDENT>if callable(req.alpha):<EOL><INDENT>req_alpha = req.alpha(alpha)<EOL><DEDENT>else:<EOL><INDENT>req_alpha = req.alpha<EOL><DEDENT>add_func(req.func, alpha=req_alpha)<EOL>meta_deps[func].add(req.func)<EOL>meta_deps[req.func]  <EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>info = d[func]<EOL>if not needs_alpha:<EOL><INDENT>if pos is not None:<EOL><INDENT>if info.pos != [None]:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(func_name))<EOL><DEDENT>info.pos[<NUM_LIT:0>] = pos<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>try:<EOL><INDENT>idx = info.alphas.index(alpha)<EOL><DEDENT>except ValueError:<EOL><INDENT>info.alphas.append(alpha)<EOL>info.pos.append(pos)<EOL>if is_meta:<EOL><INDENT>for req in func.needs_results:<EOL><INDENT>if callable(req.alpha):<EOL><INDENT>req_alpha = req.alpha(alpha)<EOL><DEDENT>else:<EOL><INDENT>req_alpha = req.alpha<EOL><DEDENT>add_func(req.func, alpha=req_alpha)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if pos is not None:<EOL><INDENT>if info.pos[idx] is not None:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(func_name, alpha))<EOL><DEDENT>info.pos[idx] = pos<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>for i, spec in enumerate(specs):<EOL><INDENT>func_name, alpha = (spec.split('<STR_LIT::>', <NUM_LIT:1>) + [None])[:<NUM_LIT:2>]<EOL>if alpha is not None:<EOL><INDENT>alpha = float(alpha)<EOL><DEDENT>try:<EOL><INDENT>func = func_mapping[func_name]<EOL><DEDENT>except KeyError:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(func_name))<EOL><DEDENT>needs_alpha = getattr(func, '<STR_LIT>', False)<EOL>if needs_alpha and alpha is None:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(func_name, spec))<EOL><DEDENT>elif not needs_alpha and alpha is not None:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(func_name, spec))<EOL><DEDENT>add_func(func, alpha, i)<EOL><DEDENT>meta_counter = itertools.count(-<NUM_LIT:1>, step=-<NUM_LIT:1>)<EOL>for info in itertools.chain(itervalues(funcs), itervalues(metas)):<EOL><INDENT>for i, pos in enumerate(info.pos):<EOL><INDENT>if pos is None:<EOL><INDENT>info.pos[i] = next(meta_counter)<EOL><DEDENT><DEDENT><DEDENT>for func, info in iteritems(metas):<EOL><INDENT>deps = info.deps<EOL>assert deps == []<EOL>for req in func.needs_results:<EOL><INDENT>f = req.func<EOL>req_info = (metas if hasattr(f, '<STR_LIT>') else funcs)[f]<EOL>if req.alpha is not None:<EOL><INDENT>if callable(req.alpha):<EOL><INDENT>req_alpha = req.alpha(info.alphas)<EOL><DEDENT>else:<EOL><INDENT>req_alpha = req.alpha<EOL><DEDENT>find_alpha = np.vectorize(req_info.alphas.index, otypes=[int])<EOL>pos = np.asarray(req_info.pos)[find_alpha(req_alpha)]<EOL>if np.isscalar(pos):<EOL><INDENT>deps.append(pos[()])<EOL><DEDENT>else:<EOL><INDENT>deps.extend(pos)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pos, = req_info.pos<EOL>deps.append(pos)<EOL><DEDENT><DEDENT><DEDENT>meta_order = topological_sort(meta_deps)<EOL>metas_ordered = OrderedDict(<EOL>(f, metas[f]) for f in meta_order if hasattr(f, '<STR_LIT>'))<EOL>return funcs, metas_ordered, -next(meta_counter) - <NUM_LIT:1><EOL>

Set up the different functions we need to call.

Returns:
    - a dict mapping base estimator functions to _FuncInfo objects.
      If the function needs_alpha, then the alphas attribute is an array
      of alpha values and pos is a corresponding array of indices.
      Otherwise, alphas is None and pos is a list containing a single index.
      Indices are >= 0 if they correspond to something in a spec,
      and negative if they're just used for a meta estimator but not
      directly requested.
    - an OrderedDict mapping functions to _MetaFuncInfo objects.
      alphas and pos are like for _FuncInfo; deps is a list of indices
      which should be passed to the estimator. Note that these might be
      other meta functions; this list is guaranteed to be in an order
      such that all dependencies are resolved before calling that function.
      If no such order is possible, raise ValueError.
    - the number of meta-only results

# TODO: update doctests for _parse_specs

>>> _parse_specs(['renyi:.8', 'hellinger', 'renyi:.9'])
({<function alpha_div at 0x10954f848>:
        _FuncInfo(alphas=[0.8, 0.5, 0.9], pos=[-1, -2, -3])},
 OrderedDict([
    (<function hellinger at 0x10954fc80>,
        _MetaFuncInfo(alphas=None, pos=[1], deps=[array(-2)])),
    (<function renyi at 0x10954fcf8>,
        _MetaFuncInfo(alphas=[0.8, 0.9], pos=[0, 2], deps=[-1, -3]))
 ]), 3)

>>> _parse_specs(['renyi:.8', 'hellinger', 'renyi:.9', 'l2'])
({<function alpha_div at 0x10954f848>:
    _FuncInfo(alphas=[0.8, 0.5, 0.9], pos=[-1, -2, -3]),
  <function linear at 0x10954f758>: _FuncInfo(alphas=None, pos=[-4])
 }, OrderedDict([
    (<function hellinger at 0x10954fc80>,
        _MetaFuncInfo(alphas=None, pos=[1], deps=[array(-2)])),
    (<function l2 at 0x10954fde8>,
        _MetaFuncInfo(alphas=None, pos=[3], deps=[-4])),
    (<function renyi at 0x10954fcf8>,
        _MetaFuncInfo(alphas=[0.8, 0.9], pos=[0, 2], deps=[-1, -3]))
 ]), 4)

>>> _parse_specs(['renyi:.8', 'hellinger', 'renyi:.9', 'l2', 'linear'])
({<function alpha_div at 0x10954f848>:
    _FuncInfo(alphas=[0.8, 0.5, 0.9], pos=[-1, -2, -3]),
  <function linear at 0x10954f758>: _FuncInfo(alphas=None, pos=[4])
 }, OrderedDict([
    (<function hellinger at 0x10954fc80>,
        _MetaFuncInfo(alphas=None, pos=[1], deps=[array(-2)])),
    (<function l2 at 0x10954fde8>,
        _MetaFuncInfo(alphas=None, pos=[3], deps=[4])),
    (<function renyi at 0x10954fcf8>,
        _MetaFuncInfo(alphas=[0.8, 0.9], pos=[0, 2], deps=[-1, -3]))
 ]), 3)


Ks = as_integer_type(self.Ks)<EOL>if Ks.ndim != <NUM_LIT:1>:<EOL><INDENT>raise TypeError("<STR_LIT>".format(Ks.shape))<EOL><DEDENT>if Ks.min() < <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>".format(Ks.min()))<EOL><DEDENT>return Ks<EOL>


if self.n_jobs == -<NUM_LIT:1>:<EOL><INDENT>from multiprocessing import cpu_count<EOL>return cpu_count()<EOL><DEDENT>return self.n_jobs<EOL>

n_jobs, but with the number of cores instead of -1.


args = {'<STR_LIT>': self._n_jobs}<EOL>if self.flann_algorithm == '<STR_LIT>':<EOL><INDENT>if X is None or X.dim > <NUM_LIT:5>:<EOL><INDENT>args['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>args['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>args['<STR_LIT>'] = self.flann_algorithm<EOL><DEDENT>if self.flann_args:<EOL><INDENT>args.update(self.flann_args)<EOL><DEDENT>try:<EOL><INDENT>FLANNParameters().update(args)<EOL><DEDENT>except AttributeError as e:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg.format(e))<EOL><DEDENT>return args<EOL>

The dictionary of arguments to give to FLANN.


def fit(self, X, y=None, get_rhos=False):

self.features_ = X = as_features(X, stack=True, bare=True)<EOL>Ks = self._get_Ks()<EOL>_, _, _, max_K, save_all_Ks, _ = _choose_funcs(<EOL>self.div_funcs, Ks, X.dim, X.n_pts, None, self.version)<EOL>if max_K >= X.n_pts.min():<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(max_K, X.n_pts.min()))<EOL><DEDENT>memory = self.memory<EOL>if isinstance(memory, string_types):<EOL><INDENT>memory = Memory(cachedir=memory, verbose=<NUM_LIT:0>)<EOL><DEDENT>self.indices_ = id = memory.cache(_build_indices)(X, self._flann_args())<EOL>if get_rhos:<EOL><INDENT>self.rhos_ = _get_rhos(X, id, Ks, max_K, save_all_Ks, self.min_dist)<EOL><DEDENT>elif hasattr(self, '<STR_LIT>'):<EOL><INDENT>del self.rhos_<EOL><DEDENT>return self<EOL>

Sets up for divergence estimation "from" new data "to" X.
Builds FLANN indices for each bag, and maybe gets within-bag distances.

Parameters
----------
X : list of arrays or :class:`skl_groups.features.Features`
    The bags to search "to".

get_rhos : boolean, optional, default False
    Compute within-bag distances :attr:`rhos_`. These are only needed
    for some divergence functions or if do_sym is passed, and they'll
    be computed (and saved) during :meth:`transform` if they're not
    computed here.

    If you're using Jensen-Shannon divergence, a higher max_K may
    be needed once it sees the number of points in the transformed bags,
    so the computation here might be wasted.


X = as_features(X, stack=True, bare=True)<EOL>Y = self.features_<EOL>Ks = np.asarray(self.Ks)<EOL>if X.dim != Y.dim:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(Y.dim, X.dim))<EOL><DEDENT>memory = self.memory<EOL>if isinstance(memory, string_types):<EOL><INDENT>memory = Memory(cachedir=memory, verbose=<NUM_LIT:0>)<EOL><DEDENT>est = memory.cache(_est_divs, ignore=['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>output, self.rhos_ = est(<EOL>X, Y, self.indices_, getattr(self, '<STR_LIT>', None),<EOL>self.div_funcs, Ks,<EOL>self.do_sym, self.clamp, self.version, self.min_dist,<EOL>self._flann_args(), self._n_jobs)<EOL>return output<EOL>

r'''
        Computes the divergences from X to :attr:`features_`.

        Parameters
        ----------
        X : list of bag feature arrays or :class:`skl_groups.features.Features`
            The bags to search "from".

        Returns
        -------
        divs : array of shape ``[len(div_funcs), len(Ks), len(X), len(features_)] + ([2] if do_sym else [])``
            The divergences from X to :attr:`features_`.
            ``divs[d, k, i, j]`` is the ``div_funcs[d]`` divergence
            from ``X[i]`` to ``fetaures_[j]`` using a K of ``Ks[k]``.
            If ``do_sym``, ``divs[d, k, i, j, 0]`` is
            :math:`D_{d,k}( X_i \| \texttt{features_}_j)` and
            ``divs[d, k, i, j, 1]`` is :math:`D_{d,k}(\texttt{features_}_j \| X_i)`.


self._check_fitted()<EOL>return self.inds_.shape[<NUM_LIT:1>]<EOL>

The dimension of the inputs, once fitted.


self._check_fitted()<EOL>return self.inds_.shape[<NUM_LIT:0>]<EOL>

The dimension of the output vectors, once fitted.


if is_integer(X):<EOL><INDENT>dim = X<EOL><DEDENT>else:<EOL><INDENT>X = as_features(X)<EOL>dim = X.dim<EOL><DEDENT>M = self.smoothness<EOL>inds = np.mgrid[(slice(M + <NUM_LIT:1>),) * dim].reshape(dim, (M + <NUM_LIT:1>) ** dim).T<EOL>self.inds_ = inds[(inds ** <NUM_LIT:2>).sum(axis=<NUM_LIT:1>) <= M ** <NUM_LIT:2>]<EOL>return self<EOL>

Picks the elements of the basis to use for the given data.

Only depends on the dimension of X. If it's more convenient, you can
pass a single integer for X, which is the dimension to use.

Parameters
----------
X : an integer, a :class:`Features` instance, or a list of bag features
    The input data, or just its dimension, since only the dimension is
    needed here.


self._check_fitted()<EOL>M = self.smoothness<EOL>dim = self.dim_<EOL>inds = self.inds_<EOL>do_check = self.do_bounds_check<EOL>X = as_features(X)<EOL>if X.dim != dim:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ValueError(msg.format(dim, X.dim))<EOL><DEDENT>Xt = np.empty((len(X), self.inds_.shape[<NUM_LIT:0>]))<EOL>Xt.fill(np.nan)<EOL>if self.basis == '<STR_LIT>':  <EOL><INDENT>coefs = (np.pi * np.arange(M + <NUM_LIT:1>))[..., :]<EOL>for i, bag in enumerate(X):<EOL><INDENT>if do_check:<EOL><INDENT>if np.min(bag) < <NUM_LIT:0> or np.max(bag) > <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>".format(i))<EOL><DEDENT><DEDENT>phi = coefs * bag[..., np.newaxis]<EOL>np.cos(phi, out=phi)<EOL>phi[:, :, <NUM_LIT:1>:] *= np.sqrt(<NUM_LIT:2>)<EOL>B = reduce(op.mul, (phi[:, i, inds[:, i]] for i in xrange(dim)))<EOL>Xt[i, :] = np.mean(B, axis=<NUM_LIT:0>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>".format(self.basis))<EOL><DEDENT>return Xt<EOL>

Transform a list of bag features into its projection series
representation.

Parameters
----------
X : :class:`skl_groups.features.Features` or list of bag feature arrays
    New data to transform. The data should all lie in [0, 1];
    use :class:`skl_groups.preprocessing.BagMinMaxScaler` if not.

Returns
-------
X_new : integer array, shape ``[len(X), dim_]``
    X transformed into the new space.


return self<EOL>

Do nothing; this is a stateless transformer.


X = as_features(X)<EOL>return np.vstack([np.mean(bag, axis=<NUM_LIT:0>) for bag in X])<EOL>

Transform a list of bag features into a matrix of its mean features.

Parameters
----------
X : :class:`skl_groups.features.Features` or list of bag feature arrays
    Data to transform.

Returns
-------
X_new : array, shape ``[len(X), X.dim]``
    X transformed into its means.


return self.kmeans.n_clusters<EOL>


self._check_fitted()<EOL>return self.kmeans_fit_.cluster_centers_<EOL>

The selected codewords; shape [n_codewords, n_features].


self.kmeans_fit_ = copy(self.kmeans)<EOL>X = as_features(X, stack=True)<EOL>self.kmeans_fit_.fit(X.stacked_features) <EOL>return self<EOL>

Choose the codewords based on a training set.

Parameters
----------
X : :class:`skl_groups.features.Features` or list of arrays of shape ``[n_samples[i], n_features]``
    Training set. If a Features object, it will be stacked.


self._check_fitted()<EOL>X = as_features(X, stack=True)<EOL>assignments = self.kmeans_fit_.predict(X.stacked_features)<EOL>return self._group_assignments(X, assignments)<EOL>

Transform a list of bag features into its bag-of-words representation.

Parameters
----------
X : :class:`skl_groups.features.Features` or list of bag feature arrays
    New data to transform.

Returns
-------
X_new : integer array, shape [len(X), kmeans.n_clusters]
    X transformed into the new space.


X = as_features(X, stack=True)<EOL>self.kmeans_fit_ = copy(self.kmeans)<EOL>assignments = self.kmeans_fit_.fit_predict(X.stacked_features) <EOL>return self._group_assignments(X, assignments)<EOL>

Compute clustering and transform a list of bag features into its
bag-of-words representation. Like calling fit(X) and then transform(X),
but more efficient.

Parameters
----------
X : :class:`skl_groups.features.Features` or list of bag feature arrays
    New data to transform.

Returns
-------
X_new : integer array, shape [len(X), kmeans.n_clusters]
    X transformed into the new space.


return issubclass(np.asanyarray(x).dtype.type, np.integer)<EOL>

Checks whether the array is of an integral type.


ary = np.asanyarray(ary)<EOL>return is_integer_type(ary) or ary.dtype.kind == '<STR_LIT:b>'<EOL>

Checks whether the array is either integral or boolean.


return np.isscalar(x) and is_integer_type(x)<EOL>

Checks whether the argument is a single integer.


return np.isscalar(x) and is_categorical_type(x)<EOL>

Checks whether the argument is a single integer or boolean.


ary = np.asanyarray(ary)<EOL>if is_integer_type(ary):<EOL><INDENT>return ary<EOL><DEDENT>rounded = np.rint(ary)<EOL>if np.any(rounded != ary):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return rounded.astype(int)<EOL>

Returns argument as an integer array, converting floats if convertable.
Raises ValueError if it's a float array with nonintegral values.


logger = logging.getLogger(name)<EOL>logger.setLevel(logging.INFO)<EOL>logger.addHandler(ProgressBarHandler(**kwargs))<EOL>

Sets up a :class:`ProgressBarHandler` to handle progess logs for
a given module.

Parameters
----------
name : string
    The module name of the progress logger to use. For example,
    :class:`skl_groups.divergences.KNNDivergenceEstimator`
    uses ``'skl_groups.divergences.knn.progress'``.

* : anything
    Other keyword arguments are passed to the :class:`ProgressBarHandler`.


self.logger.info(json.dumps(['<STR_LIT>', self.name, total]))<EOL>

Signal the start of the process.

Parameters
----------
total : int
    The total number of steps in the process, or None if unknown.


<EOL>self.logger.info('<STR_LIT>'.format(idx))<EOL>

Update the current state.

Parameters
----------
idx : int
    The current state through the process.


self.logger.info(json.dumps(['<STR_LIT>']))<EOL>


def as_features(X, stack=False, bare=False):

if isinstance(X, Features):<EOL><INDENT>if stack:<EOL><INDENT>X.make_stacked()<EOL><DEDENT>return X.bare() if bare else X<EOL><DEDENT>return Features(X, stack=stack, bare=bare)<EOL>

Returns a version of X as a :class:`Features` object.

Parameters
----------
stack : boolean, default False
    Make a stacked version of X. Note that if X is a features object,
    this will stack it in-place, since that's usually what you want.
    (If not, just use the :class:`Features` constructor instead.)

bare : boolean, default False
    Return a bare version of X (no metadata).

Returns
-------
feats : :class:`Features`
    A version of X. If X is already a :class:`Features` object, the original
    X may be returned, depending on the arguments.


if self.stacked:<EOL><INDENT>return<EOL><DEDENT>self._boundaries = bounds = np.r_[<NUM_LIT:0>, np.cumsum(self.n_pts)]<EOL>self.stacked_features = stacked = np.vstack(self.features)<EOL>self.features = np.array(<EOL>[stacked[bounds[i-<NUM_LIT:1>]:bounds[i]] for i in xrange(<NUM_LIT:1>, len(bounds))],<EOL>dtype=object)<EOL>self.stacked = True<EOL>

If unstacked, convert to stacked. If stacked, do nothing.


return self.n_pts.sum()<EOL>


return self.features[<NUM_LIT:0>].shape[<NUM_LIT:1>]<EOL>


return self.features[<NUM_LIT:0>].dtype<EOL>


def copy(self, stack=False, copy_meta=False, memo=None):

if self.stacked:<EOL><INDENT>fs = deepcopy(self.stacked_features, memo)<EOL>n_pts = self.n_pts.copy()<EOL><DEDENT>elif stack:<EOL><INDENT>fs = np.vstack(self.features)<EOL>n_pts = self.n_pts.copy()<EOL><DEDENT>else:<EOL><INDENT>fs = deepcopy(self.features, memo)<EOL>n_pts = None<EOL><DEDENT>meta = deepcopy(self.meta, memo) if copy_meta else self.meta<EOL>return Features(fs, n_pts, copy=False, **meta)<EOL>

Copies the Feature object. Makes a copy of the features array.

Parameters
----------
stack : boolean, optional, default False
    Whether to stack the copy if this one is unstacked.

copy_meta : boolean, optional, default False
    Also copy the metadata. If False, metadata in both points to the
    same object.


if not self.meta:<EOL><INDENT>return self<EOL><DEDENT>elif self.stacked:<EOL><INDENT>return Features(self.stacked_features, self.n_pts, copy=False)<EOL><DEDENT>else:<EOL><INDENT>return Features(self.features, copy=False)<EOL><DEDENT>

Make a Features object with no metadata; points to the same features.


return check_output(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", path]).decode().strip()<EOL>

Write file at path to Git index, return its SHA1 as a string.


for f, sha1 in files:<EOL><INDENT>yield "<STR_LIT>".format(sha1, f)<EOL><DEDENT>for d, sha1 in dirs:<EOL><INDENT>yield "<STR_LIT>".format(sha1, d)<EOL><DEDENT>


dir_hash = {}<EOL>for root, dirs, files in os.walk(path, topdown=False):<EOL><INDENT>f_hash = ((f, hash_file(join(root, f))) for f in files)<EOL>d_hash = ((d, dir_hash[join(root, d)]) for d in dirs)<EOL>dir_hash[join(*split(root))] = _mktree(f_hash, d_hash)<EOL><DEDENT>return dir_hash[path]<EOL>

Write directory at path to Git index, return its SHA1 as a string.


if vor.points.shape[<NUM_LIT:1>] != <NUM_LIT:2>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>new_regions = []<EOL>new_vertices = vor.vertices.tolist()<EOL>center = vor.points.mean(axis=<NUM_LIT:0>)<EOL>if radius is None:<EOL><INDENT>radius = vor.points.ptp().max()*<NUM_LIT:2><EOL><DEDENT>all_ridges = {}<EOL>for (p1, p2), (v1, v2) in zip(vor.ridge_points, vor.ridge_vertices):<EOL><INDENT>all_ridges.setdefault(p1, []).append((p2, v1, v2))<EOL>all_ridges.setdefault(p2, []).append((p1, v1, v2))<EOL><DEDENT>for p1, region in enumerate(vor.point_region):<EOL><INDENT>vertices = vor.regions[region]<EOL>if all(v >= <NUM_LIT:0> for v in vertices):<EOL><INDENT>new_regions.append(vertices)<EOL>continue<EOL><DEDENT>ridges = all_ridges[p1]<EOL>new_region = [v for v in vertices if v >= <NUM_LIT:0>]<EOL>for p2, v1, v2 in ridges:<EOL><INDENT>if v2 < <NUM_LIT:0>:<EOL><INDENT>v1, v2 = v2, v1<EOL><DEDENT>if v1 >= <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>t = vor.points[p2] - vor.points[p1] <EOL>t /= np.linalg.norm(t)<EOL>n = np.array([-t[<NUM_LIT:1>], t[<NUM_LIT:0>]])  <EOL>midpoint = vor.points[[p1, p2]].mean(axis=<NUM_LIT:0>)<EOL>direction = np.sign(np.dot(midpoint - center, n)) * n<EOL>far_point = vor.vertices[v2] + direction * radius<EOL>new_region.append(len(new_vertices))<EOL>new_vertices.append(far_point.tolist())<EOL><DEDENT>vs = np.asarray([new_vertices[v] for v in new_region])<EOL>c = vs.mean(axis=<NUM_LIT:0>)<EOL>angles = np.arctan2(vs[:,<NUM_LIT:1>] - c[<NUM_LIT:1>], vs[:,<NUM_LIT:0>] - c[<NUM_LIT:0>])<EOL>new_region = np.array(new_region)[np.argsort(angles)]<EOL>new_regions.append(new_region.tolist())<EOL><DEDENT>return new_regions, np.asarray(new_vertices)<EOL>

Reconstruct infinite voronoi regions in a 2D diagram to finite
regions.

Parameters
----------
vor : Voronoi
    Input diagram
radius : float, optional
    Distance to 'points at infinity'.

Returns
-------
regions : list of tuples
    Indices of vertices in each revised Voronoi regions.
vertices : list of tuples
    Coordinates for revised Voronoi vertices. Same as coordinates
    of input vertices, with 'points at infinity' appended to the
    end.


result = first_name()<EOL>if with_num:<EOL><INDENT>result += str(random.randint(<NUM_LIT>, <NUM_LIT>))<EOL><DEDENT>return result.lower()<EOL>

Return a random user name.

    Basically it's lowercased result of
    :py:func:`~forgery_py.forgery.name.first_name()` with a number appended
    if `with_num`.


return random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>


result = random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>result += '<STR_LIT:.>' + top_level_domain()<EOL>return result.lower()<EOL>

Return a random domain name.

    Lowercased result of :py:func:`~forgery_py.forgery.name.company_name()`
    plus :py:func:`~top_level_domain()`.


if not user:<EOL><INDENT>user = user_name()<EOL><DEDENT>else:<EOL><INDENT>user = user.strip().replace('<STR_LIT:U+0020>', '<STR_LIT:_>').lower()<EOL><DEDENT>return user + '<STR_LIT:@>' + domain_name()<EOL>

Return random e-mail address in a hopefully imaginary domain.

    If `user` is ``None`` :py:func:`~user_name()` will be used. Otherwise it
    will be lowercased and will have spaces replaced with ``_``.

    Domain name is created using :py:func:`~domain_name()`.


return lorem_ipsum.title(words_quantity=words_quantity)<EOL>

An alias for lorem_ipsum.title(words_quantity)


return '<STR_LIT:.>'.join([str(random.randint(<NUM_LIT:0>, <NUM_LIT:255>)) for _ in range(<NUM_LIT:0>, <NUM_LIT:4>)])<EOL>


<EOL>magnitude = <NUM_LIT:16> ** <NUM_LIT:4><EOL>return "<STR_LIT::>".join(("<STR_LIT>" % random.randint(<NUM_LIT:0>, magnitude) for _ in range(<NUM_LIT:8>)))<EOL>

Return a random IPv6 address for Python versions prior to 3.3.


try:<EOL><INDENT>import ipaddress<EOL><DEDENT>except ImportError:<EOL><INDENT>return _py2_ip_v6()<EOL><DEDENT>return str(ipaddress.IPv6Address(random.randint(<NUM_LIT:0>, <NUM_LIT:2>**<NUM_LIT>)))<EOL>


return words(quantity=<NUM_LIT:1>)<EOL>


global _words<EOL>if not _words:<EOL><INDENT>_words = '<STR_LIT:U+0020>'.join(get_dictionary('<STR_LIT>')).lower().replace('<STR_LIT:\n>', '<STR_LIT>')<EOL>_words = re.sub(r'<STR_LIT>', '<STR_LIT>', _words)<EOL>_words = _words.split('<STR_LIT:U+0020>')<EOL><DEDENT>result = random.sample(_words, quantity)<EOL>if as_list:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:U+0020>'.join(result)<EOL><DEDENT>


result = words(quantity=words_quantity)<EOL>result += random.choice('<STR_LIT>')<EOL>return result.capitalize()<EOL>

Return a random sentence to be used as e.g. an e-mail subject.


return sentences(quantity=<NUM_LIT:1>)<EOL>


result = [sntc.strip() for sntc in<EOL>random.sample(get_dictionary('<STR_LIT>'), quantity)]<EOL>if as_list:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:U+0020>'.join(result)<EOL><DEDENT>


def paragraph(separator='<STR_LIT>', wrap_start='<STR_LIT>', wrap_end='<STR_LIT>',<EOL>html=False, sentences_quantity=<NUM_LIT:3>):

return paragraphs(quantity=<NUM_LIT:1>, separator=separator, wrap_start=wrap_start,<EOL>wrap_end=wrap_end, html=html,<EOL>sentences_quantity=sentences_quantity)<EOL>


def paragraphs(quantity=<NUM_LIT:2>, separator='<STR_LIT>', wrap_start='<STR_LIT>', wrap_end='<STR_LIT>',<EOL>html=False, sentences_quantity=<NUM_LIT:3>, as_list=False):

if html:<EOL><INDENT>wrap_start = '<STR_LIT>'<EOL>wrap_end = '<STR_LIT>'<EOL>separator = '<STR_LIT>'<EOL><DEDENT>result = []<EOL>try:<EOL><INDENT>for _ in xrange(<NUM_LIT:0>, quantity):<EOL><INDENT>result.append(wrap_start +<EOL>sentences(sentences_quantity) +<EOL>wrap_end)<EOL><DEDENT><DEDENT>except NameError:<EOL><INDENT>for _ in range(<NUM_LIT:0>, quantity):<EOL><INDENT>result.append(wrap_start +<EOL>sentences(sentences_quantity) +<EOL>wrap_end)<EOL><DEDENT><DEDENT>if as_list:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>return separator.join(result)<EOL><DEDENT>


s = re.sub(r'<STR_LIT:\n>', '<STR_LIT:U+0020>',  s.lower())<EOL>return re.sub(r'<STR_LIT>', '<STR_LIT>', s)<EOL>

Return a lowercased string with non alphabetic chars removed.

    White spaces are not to be removed.


line = map(_to_lower_alpha_only,<EOL>'<STR_LIT>'.join(random.sample(get_dictionary('<STR_LIT>'), quantity)))<EOL>return '<STR_LIT>'.join(line)[:quantity]<EOL>


return characters(quantity=<NUM_LIT:1>)<EOL>


def text(what="<STR_LIT>", *args, **kwargs):

if what == "<STR_LIT>":<EOL><INDENT>return character(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return characters(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return word(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return words(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return sentence(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return sentences(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return paragraph(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT>":<EOL><INDENT>return paragraphs(*args, **kwargs)<EOL><DEDENT>elif what == "<STR_LIT:title>":<EOL><INDENT>return title(*args, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>raise NameError('<STR_LIT>')<EOL><DEDENT>

An aggregator for all above defined public methods.


return _to_lower_alpha_only('<STR_LIT>'.join(get_dictionary('<STR_LIT>')))<EOL>

Return a whole lorem_ipsum dictionary as a lowercase string.


return lorem_ipsum_characters().strip().split('<STR_LIT:U+0020>')<EOL>

Return a list of all lowercased words.

    Words are taken from the `lorem_ipsum dictionary`.


_dict = get_dictionary('<STR_LIT>')<EOL>_dict += get_dictionary('<STR_LIT>')<EOL>return random.choice(_dict).strip()<EOL>

Return a random male of female first name.


return random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>


return first_name() + '<STR_LIT:U+0020>' + last_name()<EOL>

Return a random full name.

    Equivalent of ``first_name() + ' ' + last_name()``.


result = random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>result = result.replace('<STR_LIT>', job_title_suffix())<EOL>return result<EOL>


Return a random name title, e.g. ``Mr``.


Return a random name suffix, e.g. ``Jr``.


Return a random location name, e.g. ``MI6 Headquarters``.


if abbr:<EOL><INDENT>return random.choice(DAYS_ABBR)<EOL><DEDENT>else:<EOL><INDENT>return random.choice(DAYS)<EOL><DEDENT>

Return a random (abbreviated if `abbr`) day of week name.


if numerical:<EOL><INDENT>return random.randint(<NUM_LIT:1>, <NUM_LIT:12>)<EOL><DEDENT>else:<EOL><INDENT>if abbr:<EOL><INDENT>return random.choice(MONTHS_ABBR)<EOL><DEDENT>else:<EOL><INDENT>return random.choice(MONTHS)<EOL><DEDENT><DEDENT>

Return a random (abbreviated if `abbr`) month name or month number if
    `numerical`.


def year(past=False, min_delta=<NUM_LIT:0>, max_delta=<NUM_LIT:20>):

return dt.date.today().year + _delta(past, min_delta, max_delta)<EOL>


return random.randint(<NUM_LIT:1>, month_length)<EOL>

Return a random day number in a `month_length` days long month.


def date(past=False, min_delta=<NUM_LIT:0>, max_delta=<NUM_LIT:20>):

timedelta = dt.timedelta(days=_delta(past, min_delta, max_delta))<EOL>return dt.date.today() + timedelta<EOL>

Return a random `dt.date` object. Delta args are days.


def datetime(past=False, min_delta=<NUM_LIT:0>, max_delta=<NUM_LIT:20>):

timedelta = dt.timedelta(days=_delta(past, min_delta, max_delta))<EOL>return dt.datetime.today() + timedelta<EOL>

Return a random `dt.dt` object. Delta args are days.


value = random.choice(range(min * <NUM_LIT:100>, max * <NUM_LIT:100>))<EOL>return "<STR_LIT>" % (float(value) / <NUM_LIT:100>)<EOL>

Return a str of decimal with two digits after a decimal mark.


return "<STR_LIT>" % float(money(min=min, max=max))<EOL>

Return a random sum of money with a dollar sign as a prefix.


return random.uniform(<NUM_LIT:0.0>, <NUM_LIT:1.0>) * <NUM_LIT> - <NUM_LIT><EOL>

Return a random latitude in the range of [-90.0, +90.0].

    Latitude is a float.


return random.randint(<NUM_LIT:0>, <NUM_LIT>) - <NUM_LIT><EOL>

Return a random latitude's degrees component in the range [-180, +180].

    Latitude's degree is an int.


return random.choice(['<STR_LIT:N>', '<STR_LIT:S>'])<EOL>

Return a random a latitude's direction component.

    Latitude's direction is denoted as either "N" (north) or "S" (south).


return random.randint(<NUM_LIT:0>, <NUM_LIT> - <NUM_LIT:1>)<EOL>

Return a random latitude's minutes component in the range [0, 60).

    Latitude's minutes is an int.


Return a random latitude's seconds component in the range [0, 60).

    Latitude's seconds is an int.


Return a random longitude in the range [-180.0, +180.0].

    Longitude is a float.


return latitude_degrees()<EOL>

Return a random longitude's degrees component in the range [-180, +180].

    Longitude's degrees is an int.


return random.choice(['<STR_LIT:E>', '<STR_LIT>'])<EOL>

Return a random longitude's direction component.

    Longitude's direction is denoted as either "E" (east) or "W" (west).


return latitude_minutes()<EOL>

Return a random longitude's minutes component in the range [0, 60).

    longitude's minutes is an int.


return latitude_seconds()<EOL>

Return a random longitude's seconds component in the range [0, 60).

    Longitude's seconds is an int.


Return a random currency description, e.g. `United Kingdom Pounds`.


Return a random currency code, e.g. `GBP`.


return '<STR_LIT>'.join(random.sample(HEX_DIGITS, <NUM_LIT:6>))<EOL>


return '<STR_LIT>'.join(random.sample(HEX_DIGITS, <NUM_LIT:3>))<EOL>

Return random short HEX color (e.g. `FFF` color).


def text(length=None, at_least=<NUM_LIT:10>, at_most=<NUM_LIT:15>, lowercase=True,<EOL>uppercase=True, digits=True, spaces=True, punctuation=False):

base_string = '<STR_LIT>'<EOL>if lowercase:<EOL><INDENT>base_string += string.ascii_lowercase<EOL><DEDENT>if uppercase:<EOL><INDENT>base_string += string.ascii_uppercase<EOL><DEDENT>if digits:<EOL><INDENT>base_string += string.digits<EOL><DEDENT>if spaces:<EOL><INDENT>base_string += '<STR_LIT:U+0020>'<EOL><DEDENT>if punctuation:<EOL><INDENT>base_string += string.punctuation<EOL><DEDENT>if len(base_string) == <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if not length:<EOL><INDENT>length = random.randint(at_least, at_most)<EOL><DEDENT>result = '<STR_LIT>'<EOL>try:<EOL><INDENT>for _ in xrange(<NUM_LIT:0>, length):<EOL><INDENT>result += random.choice(base_string)<EOL><DEDENT><DEDENT>except NameError:<EOL><INDENT>for i in range(<NUM_LIT:0>, length):<EOL><INDENT>result += random.choice(base_string)<EOL><DEDENT><DEDENT>return result<EOL>

Return random text.

If `length` is present the text will be exactly this chars long. Else the
text will be something between `at_least` and `at_most` chars long.


return random.choice(BOOLEAN)<EOL>


return random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>


if not salt:<EOL><INDENT>salt = str(datetime.utcnow())<EOL><DEDENT>try:<EOL><INDENT>dk = hashlib.pbkdf2_hmac('<STR_LIT>', password.encode(), salt.encode(), <NUM_LIT>)<EOL>hexdigest = binascii.hexlify(dk).decode('<STR_LIT:utf-8>')<EOL><DEDENT>except AttributeError:<EOL><INDENT>dk = hashlib.sha1()<EOL>dk.update(password.encode() + salt.encode())<EOL>hexdigest = dk.hexdigest()<EOL><DEDENT>return hexdigest<EOL>

Return SHA1 hexdigest of a password (optionally salted with a string).


Return random frequency rate.

    Frequency rate is taken from the `frequencies` dictionary.


return random.choice(range(at_least, at_most))<EOL>

Return a random number in the range specified.


def password(at_least=<NUM_LIT:6>, at_most=<NUM_LIT:12>, lowercase=True,<EOL>uppercase=True, digits=True, spaces=False, punctuation=False):

return text(at_least=at_least, at_most=at_most, lowercase=lowercase,<EOL>uppercase=uppercase, digits=digits, spaces=spaces,<EOL>punctuation=punctuation)<EOL>

Return a random string for use as a password.


return random.choice(list(CARDS.keys()))<EOL>


sum = <NUM_LIT:0><EOL>digits = str(num)[:-<NUM_LIT:1>][::-<NUM_LIT:1>]<EOL>for i, n in enumerate(digits):<EOL><INDENT>if (i + <NUM_LIT:1>) % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>digit = int(n) * <NUM_LIT:2><EOL>if digit > <NUM_LIT:9>:<EOL><INDENT>sum += (digit - <NUM_LIT:9>)<EOL><DEDENT>else:<EOL><INDENT>sum += digit<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sum += int(n)<EOL><DEDENT><DEDENT>return ((divmod(sum, <NUM_LIT:10>)[<NUM_LIT:0>] + <NUM_LIT:1>) * <NUM_LIT:10> - sum) % <NUM_LIT:10><EOL>

Return a check digit of the given credit card number.

    Check digit calculated using Luhn algorithm ("modulus 10")
    See: http://www.darkcoding.net/credit-card/luhn-formula/


def number(type=None, length=None, prefixes=None):

<EOL>if type and type in CARDS:<EOL><INDENT>card = type<EOL><DEDENT>else:<EOL><INDENT>card = random.choice(list(CARDS.keys()))<EOL><DEDENT>if not prefixes:<EOL><INDENT>prefixes = CARDS[card]['<STR_LIT>']<EOL><DEDENT>prefix = random.choice(prefixes)<EOL>if not length:<EOL><INDENT>length = CARDS[card]['<STR_LIT>']<EOL><DEDENT>result = str(prefix)<EOL>for d in range(length - len(str(prefix))):<EOL><INDENT>result += str(basic.number())<EOL><DEDENT>last_digit = check_digit(int(result))<EOL>return int(result[:-<NUM_LIT:1>] + str(last_digit))<EOL>

Return a random credit card number.

:param type: credit card type. Defaults to a random selection.
:param length: length of the credit card number.
               Defaults to the length for the selected card type.
:param prefixes: allowed prefixes for the card number.
                 Defaults to prefixes for the selected card type.
:return: credit card randomly generated number (int)


account = [random.randint(<NUM_LIT:1>, <NUM_LIT:9>) for _ in range(<NUM_LIT:20>)]<EOL>return "<STR_LIT>".join(map(str, account))<EOL>


return '<STR_LIT>' +'<STR_LIT>'.join([str(random.randint(<NUM_LIT:1>, <NUM_LIT:9>)) for _ in range(<NUM_LIT:5>)]) +str(random.randint(<NUM_LIT:0>, <NUM_LIT>) + <NUM_LIT:50>)<EOL>

Return a random bank identification number.


if (type in TYPES) and type == '<STR_LIT>':<EOL><INDENT>return person_inn()<EOL><DEDENT>else:<EOL><INDENT>return legal_inn()<EOL><DEDENT>

Return a random taxation ID number for either a person or a company.

    Further information on the topic can be found in [1] (in russian).
    [1]: https://ru.wikipedia.org/wiki/Идентификационный_номер_налогоплательщика


mask = [<NUM_LIT:2>, <NUM_LIT:4>, <NUM_LIT:10>, <NUM_LIT:3>, <NUM_LIT:5>, <NUM_LIT:9>, <NUM_LIT:4>, <NUM_LIT:6>, <NUM_LIT:8>]<EOL>inn = [random.randint(<NUM_LIT:1>, <NUM_LIT:9>) for _ in range(<NUM_LIT:10>)]<EOL>weighted = [v * mask[i] for i, v in enumerate(inn[:-<NUM_LIT:1>])]<EOL>inn[<NUM_LIT:9>] = sum(weighted) % <NUM_LIT:11> % <NUM_LIT:10><EOL>return "<STR_LIT>".join(map(str, inn))<EOL>

Return a random taxation ID number for a company.


ogrn = "<STR_LIT>".join(map(str, [random.randint(<NUM_LIT:1>, <NUM_LIT:9>) for _ in range(<NUM_LIT:12>)]))<EOL>ogrn += str((int(ogrn) % <NUM_LIT:11> % <NUM_LIT:10>))<EOL>return ogrn<EOL>

Return a random government registration ID for a company.


if (type in TYPES) and type == '<STR_LIT>':<EOL><INDENT>return person_ogrn()<EOL><DEDENT>else:<EOL><INDENT>return legal_ogrn()<EOL><DEDENT>

Return a random government registration ID for either a person or a company.

    Further information on the topic can be found in [1] (in russian).
    [1]: https://ru.wikipedia.org/wiki/Основной_государственный_регистрационный_номер


mask11 = [<NUM_LIT:7>, <NUM_LIT:2>, <NUM_LIT:4>, <NUM_LIT:10>, <NUM_LIT:3>, <NUM_LIT:5>, <NUM_LIT:9>, <NUM_LIT:4>, <NUM_LIT:6>, <NUM_LIT:8>]<EOL>mask12 = [<NUM_LIT:3>, <NUM_LIT:7>, <NUM_LIT:2>, <NUM_LIT:4>, <NUM_LIT:10>, <NUM_LIT:3>, <NUM_LIT:5>, <NUM_LIT:9>, <NUM_LIT:4>, <NUM_LIT:6>, <NUM_LIT:8>]<EOL>inn = [random.randint(<NUM_LIT:1>, <NUM_LIT:9>) for _ in range(<NUM_LIT:12>)]<EOL>weighted11 = [v * mask11[i] for i, v in enumerate(inn[:-<NUM_LIT:2>])]<EOL>inn[<NUM_LIT:10>] = sum(weighted11) % <NUM_LIT:11> % <NUM_LIT:10><EOL>weighted12 = [v * mask12[i] for i, v in enumerate(inn[:-<NUM_LIT:1>])]<EOL>inn[<NUM_LIT:11>] = sum(weighted12) % <NUM_LIT:11> % <NUM_LIT:10><EOL>return "<STR_LIT>".join(map(str, inn))<EOL>

Return a random taxation ID number for a natural person.


ogrn = "<STR_LIT>".join(map(str, [random.randint(<NUM_LIT:1>, <NUM_LIT:9>) for _ in range(<NUM_LIT>)]))<EOL>ogrn += str((int(ogrn) % <NUM_LIT> % <NUM_LIT:10>))<EOL>return ogrn<EOL>

Return a random government registration ID for a person.


return internet.email_address(user=user)<EOL>

An alias for internet.email_address(user).


def body(quantity=<NUM_LIT:2>, separator='<STR_LIT>', wrap_start='<STR_LIT>', wrap_end='<STR_LIT>',<EOL>html=False, sentences_quantity=<NUM_LIT:3>, as_list=False):

return lorem_ipsum.paragraphs(quantity=quantity, separator=separator,<EOL>wrap_start=wrap_start, wrap_end=wrap_end,<EOL>html=html,<EOL>sentences_quantity=sentences_quantity,<EOL>as_list=as_list)<EOL>


return lorem_ipsum.title(words_quantity=words_quantity)<EOL>

An alias for lorem_ipsum.title(words_quantity)


return random.choice(get_dictionary('<STR_LIT>')).strip()<EOL>


return gender()[<NUM_LIT:0>:<NUM_LIT:1>]<EOL>


Return a random language name, e.g. ``Polish``.


length = int(random.choice(string.digits[<NUM_LIT:1>:<NUM_LIT:6>]))<EOL>return '<STR_LIT>'.join(random.sample(string.digits, length))<EOL>


return '<STR_LIT>' % (street_number(), street_name(), street_suffix())<EOL>

Return a random street address.

    Equivalent of ``street_number() + ' ' +
    street_name() + ' ' + street_suffix()``.


Return a random US abbreviated state name.


Return random Canadian province or territory.


Return random Canadian province or territory abbreviation.


format = '<STR_LIT>'<EOL>if random.random() >= <NUM_LIT:0.5>:<EOL><INDENT>format = '<STR_LIT>'<EOL><DEDENT>result = '<STR_LIT>'<EOL>for item in format:<EOL><INDENT>if item == '<STR_LIT:#>':<EOL><INDENT>result += str(random.randint(<NUM_LIT:0>, <NUM_LIT:9>))<EOL><DEDENT>else:<EOL><INDENT>result += item<EOL><DEDENT><DEDENT>return result<EOL>

Return a random ZIP code, either in `#####` or `#####-####` format.


format = '<STR_LIT>'<EOL>result = '<STR_LIT>'<EOL>for item in format:<EOL><INDENT>if item == '<STR_LIT:#>':<EOL><INDENT>result += str(random.randint(<NUM_LIT:0>, <NUM_LIT:9>))<EOL><DEDENT>else:<EOL><INDENT>result += item<EOL><DEDENT><DEDENT>return result<EOL>

Return a random phone number in `#-(###)###-####` format.


global dictionaries_cache<EOL>if dict_name not in dictionaries_cache:<EOL><INDENT>try:<EOL><INDENT>dictionary_file = codecs.open(<EOL>join(DICTIONARIES_PATH, dict_name), '<STR_LIT:r>', '<STR_LIT:utf-8>'<EOL>)<EOL><DEDENT>except IOError:<EOL><INDENT>None<EOL><DEDENT>else:<EOL><INDENT>dictionaries_cache[dict_name] = dictionary_file.readlines()<EOL>dictionary_file.close()<EOL><DEDENT><DEDENT>return dictionaries_cache[dict_name]<EOL>

Load a dictionary file ``dict_name`` (if it's not cached) and return its
contents as an array of strings.


return [array.tolist() for array in iterable_of_arrays]<EOL>

Converts an iterable of permutation matrices given as NumPy
    arrays into a list of lists.


try:<EOL><INDENT>parts = [int(n) for n in version.split('<STR_LIT:.>')]<EOL><DEDENT>except ValueError:<EOL><INDENT>fail('<STR_LIT>')<EOL><DEDENT>if len(parts) != <NUM_LIT:3>:<EOL><INDENT>fail('<STR_LIT>')<EOL><DEDENT>PARTS = {'<STR_LIT>': <NUM_LIT:0>, '<STR_LIT>': <NUM_LIT:1>, '<STR_LIT>': <NUM_LIT:2>}<EOL>index = PARTS[which] if which in PARTS else <NUM_LIT:2><EOL>before, middle, after = parts[:index], parts[index], parts[index + <NUM_LIT:1>:]<EOL>middle += <NUM_LIT:1><EOL>return '<STR_LIT:.>'.join(str(n) for n in before + [middle] + after)<EOL>

Returns the result of incrementing `version`.

    If `which` is not specified, the "patch" part of the version number will be
    incremented.  If `which` is specified, it must be ``'major'``, ``'minor'``,
    or ``'patch'``. If it is one of these three strings, the corresponding part
    of the version number will be incremented instead of the patch number.

    Returns a string representing the next version number.

    Example::

        >>> bump_version('2.7.1')
        '2.7.2'
        >>> bump_version('2.7.1', 'minor')
        '2.8.0'
        >>> bump_version('2.7.1', 'major')
        '3.0.0'


with open(filename) as f:<EOL><INDENT>match = re.search(r"<STR_LIT>" % pattern, f.read())<EOL><DEDENT>if match:<EOL><INDENT>before, version, after = match.groups()<EOL>return version<EOL><DEDENT>fail('<STR_LIT>'.format(pattern, filename))<EOL>

Gets the current version from the specified file.

    This function assumes the file includes a string of the form::

        <pattern> = <version>


Popen([sys.executable, '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']).wait()<EOL>

Uses Python's setup.py commands to build the package and upload it to
    PyPI.


print('<STR_LIT>', message, file=sys.stderr)<EOL>sys.exit(exit_status or <NUM_LIT:1>)<EOL>

Prints the specified message and exits the program with the specified
    exit status.