with self.ARTHUR_FEED_LOCK:<EOL><INDENT>if (time.time() - self.ARTHUR_LAST_MEMORY_CHECK) > <NUM_LIT:5> * self.ARTHUR_LAST_MEMORY_CHECK_TIME:<EOL><INDENT>self.ARTHUR_LAST_MEMORY_CHECK = time.time()<EOL>logger.debug("<STR_LIT>")<EOL>try:<EOL><INDENT>memory_size = self.measure_memory(self.arthur_items) / (<NUM_LIT> * <NUM_LIT>)<EOL><DEDENT>except RuntimeError as ex:<EOL><INDENT>logger.warning("<STR_LIT>", ex)<EOL>memory_size = self.ARTHUR_LAST_MEMORY_SIZE<EOL><DEDENT>self.ARTHUR_LAST_MEMORY_CHECK_TIME = time.time() - self.ARTHUR_LAST_MEMORY_CHECK<EOL>logger.debug("<STR_LIT>",<EOL>memory_size, self.ARTHUR_LAST_MEMORY_CHECK_TIME)<EOL>self.ARTHUR_LAST_MEMORY_SIZE = memory_size<EOL><DEDENT>if self.ARTHUR_LAST_MEMORY_SIZE > self.ARTHUR_MAX_MEMORY_SIZE:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>db_url = self.config.get_conf()['<STR_LIT>']['<STR_LIT>']<EOL>conn = redis.StrictRedis.from_url(db_url)<EOL>logger.debug("<STR_LIT>", db_url)<EOL>pipe = conn.pipeline()<EOL>pipe.lrange(Q_STORAGE_ITEMS, <NUM_LIT:0>, self.ARTHUR_REDIS_ITEMS - <NUM_LIT:1>)<EOL>pipe.ltrim(Q_STORAGE_ITEMS, self.ARTHUR_REDIS_ITEMS, -<NUM_LIT:1>)<EOL>items = pipe.execute()[<NUM_LIT:0>]<EOL>for item in items:<EOL><INDENT>arthur_item = pickle.loads(item)<EOL>if arthur_item['<STR_LIT>'] not in self.arthur_items:<EOL><INDENT>self.arthur_items[arthur_item['<STR_LIT>']] = []<EOL><DEDENT>self.arthur_items[arthur_item['<STR_LIT>']].append(arthur_item)<EOL><DEDENT>for tag in self.arthur_items:<EOL><INDENT>if self.arthur_items[tag]:<EOL><INDENT>logger.debug("<STR_LIT>", tag, len(self.arthur_items[tag]))<EOL><DEDENT><DEDENT><DEDENT>

Feed Ocean with backend data collected from arthur redis queue


<EOL>self.__feed_arthur()<EOL>tag = self.backend_tag(repo)<EOL>logger.debug("<STR_LIT>", self.arthur_items.keys())<EOL>logger.debug("<STR_LIT>", tag)<EOL>if tag in self.arthur_items:<EOL><INDENT>logger.debug("<STR_LIT>", tag)<EOL>while self.arthur_items[tag]:<EOL><INDENT>yield self.arthur_items[tag].pop()<EOL><DEDENT><DEDENT>


def __create_arthur_json(self, repo, backend_args):

backend_args = self._compose_arthur_params(self.backend_section, repo)<EOL>if self.backend_section == '<STR_LIT>':<EOL><INDENT>backend_args['<STR_LIT>'] = os.path.join(self.REPOSITORY_DIR, repo)<EOL><DEDENT>backend_args['<STR_LIT>'] = self.backend_tag(repo)<EOL>ajson = {"<STR_LIT>": [{}]}<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = self.backend_tag(repo)<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = self.backend_section.split("<STR_LIT::>")[<NUM_LIT:0>]<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = backend_args<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = backend_args['<STR_LIT>']<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = {}<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = {"<STR_LIT>": self.ARTHUR_TASK_DELAY}<EOL>es_col_url = self._get_collection_url()<EOL>es_index = self.conf[self.backend_section]['<STR_LIT>']<EOL>es = ElasticSearch(es_col_url, es_index)<EOL>connector = get_connector_from_name(self.backend_section)<EOL>klass = connector[<NUM_LIT:0>]  <EOL>signature = inspect.signature(klass.fetch)<EOL>last_activity = None<EOL>filter_ = {"<STR_LIT:name>": "<STR_LIT>", "<STR_LIT:value>": backend_args['<STR_LIT>']}<EOL>if '<STR_LIT>' in signature.parameters:<EOL><INDENT>last_activity = es.get_last_item_field('<STR_LIT>', [filter_])<EOL>if last_activity:<EOL><INDENT>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>']['<STR_LIT>'] = last_activity.isoformat()<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in signature.parameters:<EOL><INDENT>last_activity = es.get_last_item_field('<STR_LIT>', [filter_])<EOL>if last_activity:<EOL><INDENT>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>']['<STR_LIT>'] = last_activity<EOL><DEDENT><DEDENT>if last_activity:<EOL><INDENT>logging.info("<STR_LIT>", last_activity)<EOL><DEDENT>return(ajson)<EOL>

Create the JSON for configuring arthur to collect data

        https://github.com/grimoirelab/arthur#adding-tasks
        Sample for git:

        {
        "tasks": [
            {
                "task_id": "arthur.git",
                "backend": "git",
                "backend_args": {
                    "gitpath": "/tmp/arthur_git/",
                    "uri": "https://github.com/grimoirelab/arthur.git"
                },
                "category": "commit",
                "archive_args": {
                    "archive_path": '/tmp/test_archives',
                    "fetch_from_archive": false,
                    "archive_after": None
                },
                "scheduler_args": {
                    "delay": 10
                }
            }
        ]
        }


if uri.split('<STR_LIT:/>')[<NUM_LIT:2>] == '<STR_LIT>':<EOL><INDENT>return True<EOL><DEDENT>elif uri.split('<STR_LIT:/>')[<NUM_LIT:2>] == '<STR_LIT>':<EOL><INDENT>if uri.split('<STR_LIT:/>')[<NUM_LIT:5>] == '<STR_LIT>':<EOL><INDENT>return True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise GithubFileNotFound('<STR_LIT>' % uri)<EOL><DEDENT>

Checks the URI looks like a RAW uri in github:

        - 'https://raw.githubusercontent.com/github/hubot/master/README.md'
        - 'https://github.com/github/hubot/raw/master/README.md'

        :param uri: uri of the file


logger.debug("<STR_LIT>" % (uri))<EOL>self.__check_looks_like_uri(uri)<EOL>try:<EOL><INDENT>req = urllib.request.Request(uri)<EOL>req.add_header('<STR_LIT>', '<STR_LIT>' % self.token)<EOL>r = urllib.request.urlopen(req)<EOL><DEDENT>except urllib.error.HTTPError as err:<EOL><INDENT>if err.code == <NUM_LIT>:<EOL><INDENT>raise GithubFileNotFound('<STR_LIT>' % uri)<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return r.read().decode("<STR_LIT:utf-8>")<EOL>

Reads the file from Github

        :param uri: URI of the Github raw File

        :returns: UTF-8 text with the content


return True<EOL>

Returns True if the Task is executed per backend.
i.e. SortingHat unify is not executed per backend.


try:<EOL><INDENT>res = self.grimoire_con.get(url)<EOL>res.raise_for_status()<EOL>major = res.json()['<STR_LIT:version>']['<STR_LIT>'].split("<STR_LIT:.>")[<NUM_LIT:0>]<EOL><DEDENT>except Exception:<EOL><INDENT>logger.error("<STR_LIT>" + url)<EOL>raise<EOL><DEDENT>return major<EOL>

Get Elasticsearch version.

        Get the version of Elasticsearch. This is useful because
        Elasticsearch and Kibiter are paired (same major version for 5, 6).

        :param url: Elasticseearch url hosting Kibiter indices
        :returns:   major version, as string


self.config = config<EOL>self.conf = config.get_conf()<EOL>self.grimoire_con = grimoire_con(conn_retries=<NUM_LIT:12>)<EOL>


self.execute_batch_tasks(tasks_cls)<EOL>

Just a wrapper to the execute_batch_tasks method


self.execute_batch_tasks(tasks_cls,<EOL>self.conf['<STR_LIT>']['<STR_LIT>'],<EOL>self.conf['<STR_LIT>']['<STR_LIT>'], False)<EOL>


def execute_batch_tasks(self, tasks_cls, big_delay=<NUM_LIT:0>, small_delay=<NUM_LIT:0>, wait_for_threads=True):

def _split_tasks(tasks_cls):<EOL><INDENT>"""<STR_LIT>"""<EOL>backend_t = []<EOL>global_t = []<EOL>for t in tasks_cls:<EOL><INDENT>if t.is_backend_task(t):<EOL><INDENT>backend_t.append(t)<EOL><DEDENT>else:<EOL><INDENT>global_t.append(t)<EOL><DEDENT><DEDENT>return backend_t, global_t<EOL><DEDENT>backend_tasks, global_tasks = _split_tasks(tasks_cls)<EOL>logger.debug('<STR_LIT>' % (backend_tasks))<EOL>logger.debug('<STR_LIT>' % (global_tasks))<EOL>threads = []<EOL>stopper = threading.Event()<EOL>if len(backend_tasks) > <NUM_LIT:0>:<EOL><INDENT>repos_backend = self._get_repos_by_backend()<EOL>for backend in repos_backend:<EOL><INDENT>t = TasksManager(backend_tasks, backend, stopper, self.config, small_delay)<EOL>threads.append(t)<EOL>t.start()<EOL><DEDENT><DEDENT>if len(global_tasks) > <NUM_LIT:0>:<EOL><INDENT>gt = TasksManager(global_tasks, "<STR_LIT>", stopper, self.config, big_delay)<EOL>threads.append(gt)<EOL>gt.start()<EOL>if big_delay > <NUM_LIT:0>:<EOL><INDENT>when = datetime.now() + timedelta(seconds=big_delay)<EOL>when_str = when.strftime('<STR_LIT>')<EOL>logger.info("<STR_LIT>" % (global_tasks, when_str))<EOL><DEDENT><DEDENT>if wait_for_threads:<EOL><INDENT>time.sleep(<NUM_LIT:1>)  <EOL>stopper.set()  <EOL><DEDENT>for t in threads:<EOL><INDENT>t.join()<EOL><DEDENT>self.__check_queue_for_errors()<EOL>logger.debug("<STR_LIT>")<EOL>

Start a task manager per backend to complete the tasks.

:param task_cls: list of tasks classes to be executed
:param big_delay: seconds before global tasks are executed, should be days usually
:param small_delay: seconds before backend tasks are executed, should be minutes
:param wait_for_threads: boolean to set when threads are infinite or
                        should be synchronized in a meeting point


if self.conf['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>tasks_cls = [TaskPanels, TaskPanelsMenu]<EOL>self.execute_tasks(tasks_cls)<EOL><DEDENT>if self.conf['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>tasks_cls = [TaskInitSortingHat]<EOL>self.execute_tasks(tasks_cls)<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>tasks_cls = [TaskProjects]<EOL>self.execute_tasks(tasks_cls)<EOL>logger.info("<STR_LIT>")<EOL>return<EOL>


This method defines the workflow of SirMordred. So it calls to:
- initialize the databases
- execute the different phases for the first iteration
  (collection, identities, enrichment)
- start the collection and enrichment in parallel by data source
- start also the Sorting Hat merge


mbox_archives = '<STR_LIT>'<EOL>mailing_lists_projects = [project for project in projects if '<STR_LIT>' in projects[project]]<EOL>for mailing_lists in mailing_lists_projects:<EOL><INDENT>projects[mailing_lists]['<STR_LIT>'] = []<EOL>for mailing_list in projects[mailing_lists]['<STR_LIT>']:<EOL><INDENT>if '<STR_LIT>' in mailing_list:<EOL><INDENT>name = mailing_list.split('<STR_LIT>')[<NUM_LIT:1>]<EOL><DEDENT>elif '<STR_LIT>' in mailing_list:<EOL><INDENT>name = mailing_list.split('<STR_LIT>')[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>name = mailing_list.split('<STR_LIT:@>')[<NUM_LIT:0>]<EOL><DEDENT>list_new = "<STR_LIT>" % (name, mbox_archives, name, name)<EOL>projects[mailing_lists]['<STR_LIT>'].append(list_new)<EOL><DEDENT><DEDENT>return projects<EOL>

Compose projects.json only for mbox, but using the mailing_lists lists

    change: 'https://dev.eclipse.org/mailman/listinfo/emft-dev'
    to: 'emfg-dev /home/bitergia/mboxes/emft-dev.mbox/emft-dev.mbox

    :param projects: projects.json
    :return: projects.json with mbox


git_projects = [project for project in projects if '<STR_LIT>' in projects[project]]<EOL>for project in git_projects:<EOL><INDENT>repos = [repo for repo in projects[project]['<STR_LIT>'] if '<STR_LIT>' in repo]<EOL>if len(repos) > <NUM_LIT:0>:<EOL><INDENT>projects[project]['<STR_LIT>'] = []<EOL><DEDENT>for repo in repos:<EOL><INDENT>gerrit_project = repo.replace("<STR_LIT>", "<STR_LIT>")<EOL>gerrit_project = gerrit_project.replace("<STR_LIT>", "<STR_LIT>")<EOL>projects[project]['<STR_LIT>'].append("<STR_LIT>" + gerrit_project)<EOL><DEDENT><DEDENT>return projects<EOL>

Compose projects.json for gerrit, but using the git lists

    change: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'
    to: 'git.eclipse.org_xwt/org.eclipse.xwt

    :param projects: projects.json
    :return: projects.json with gerrit


for p in [project for project in data if len(data[project]['<STR_LIT>']) > <NUM_LIT:0>]:<EOL><INDENT>repos = []<EOL>for url in data[p]['<STR_LIT>']:<EOL><INDENT>if len(url['<STR_LIT:url>'].split()) > <NUM_LIT:1>:  <EOL><INDENT>repo = url['<STR_LIT:url>'].split()[<NUM_LIT:1>].replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>repo = url['<STR_LIT:url>'].replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>if repo not in repos:<EOL><INDENT>repos.append(repo)<EOL><DEDENT><DEDENT>projects[p]['<STR_LIT>'] = repos<EOL><DEDENT>return projects<EOL>

Compose projects.json for git

    We need to replace '/c/' by '/gitroot/' for instance

    change: 'http://git.eclipse.org/c/xwt/org.eclipse.xwt.git'
    to: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with git


for p in [project for project in data if len(data[project]['<STR_LIT>']) > <NUM_LIT:0>]:<EOL><INDENT>if '<STR_LIT>' not in projects[p]:<EOL><INDENT>projects[p]['<STR_LIT>'] = []<EOL><DEDENT>urls = [url['<STR_LIT:url>'].replace('<STR_LIT>', '<STR_LIT>') for url in data[p]['<STR_LIT>'] if<EOL>url['<STR_LIT:url>'] not in projects[p]['<STR_LIT>']]<EOL>projects[p]['<STR_LIT>'] += urls<EOL><DEDENT>for p in [project for project in data if len(data[project]['<STR_LIT>']) > <NUM_LIT:0>]:<EOL><INDENT>if '<STR_LIT>' not in projects[p]:<EOL><INDENT>projects[p]['<STR_LIT>'] = []<EOL><DEDENT>mailing_list = data[p]['<STR_LIT>']['<STR_LIT:url>'].replace('<STR_LIT>', '<STR_LIT>')<EOL>projects[p]['<STR_LIT>'].append(mailing_list)<EOL><DEDENT>return projects<EOL>

Compose projects.json for mailing lists

    At upstream has two different key for mailing list: 'mailings_lists' and 'dev_list'
    The key 'mailing_lists' is an array with mailing lists
    The key 'dev_list' is a dict with only one mailing list

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with mailing_lists


for p in [project for project in data if len(data[project]['<STR_LIT>']) > <NUM_LIT:0>]:<EOL><INDENT>if '<STR_LIT>' not in projects[p]:<EOL><INDENT>projects[p]['<STR_LIT>'] = []<EOL><DEDENT>urls = [url['<STR_LIT:url>'] for url in data[p]['<STR_LIT>'] if<EOL>url['<STR_LIT:url>'] not in projects[p]['<STR_LIT>']]<EOL>projects[p]['<STR_LIT>'] += urls<EOL><DEDENT>return projects<EOL>

Compose projects.json for github

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with github


for p in [project for project in data if len(data[project]['<STR_LIT>']) > <NUM_LIT:0>]:<EOL><INDENT>if '<STR_LIT>' not in projects[p]:<EOL><INDENT>projects[p]['<STR_LIT>'] = []<EOL><DEDENT>urls = [url['<STR_LIT>'] for url in data[p]['<STR_LIT>'] if<EOL>url['<STR_LIT>'] not in projects[p]['<STR_LIT>']]<EOL>projects[p]['<STR_LIT>'] += urls<EOL><DEDENT>return projects<EOL>

Compose projects.json for bugzilla

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with bugzilla


for project in data:<EOL><INDENT>projects[project] = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:title>': data[project]['<STR_LIT:title>']<EOL>}<EOL>}<EOL><DEDENT>return projects<EOL>

Compose the projects JSON file only with the projects name

    :param projects: projects.json
    :param data: eclipse JSON with the origin format
    :return: projects.json with titles


projects = compose_git(projects, data)<EOL>projects = compose_mailing_lists(projects, data)<EOL>projects = compose_bugzilla(projects, data)<EOL>projects = compose_github(projects, data)<EOL>projects = compose_gerrit(projects)<EOL>projects = compose_mbox(projects)<EOL>return projects<EOL>

Compose projects.json with all data sources

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with all data sources


@classmethod<EOL><INDENT>def get_repos_by_backend_section(cls, backend_section, raw=True):<DEDENT>

repos = []<EOL>projects = TaskProjects.get_projects()<EOL>for pro in projects:<EOL><INDENT>if backend_section in projects[pro]:<EOL><INDENT>if cls.GLOBAL_PROJECT not in projects:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT>else:<EOL><INDENT>if raw:<EOL><INDENT>if pro != cls.GLOBAL_PROJECT:<EOL><INDENT>if backend_section not in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT>elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[cls.GLOBAL_PROJECT][backend_section]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>not_in_unknown = [projects[pro] for pro in projects if pro != cls.GLOBAL_PROJECT][<NUM_LIT:0>]<EOL>if backend_section not in not_in_unknown:<EOL><INDENT>repos += projects[cls.GLOBAL_PROJECT][backend_section]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if pro != cls.GLOBAL_PROJECT:<EOL><INDENT>if backend_section not in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT>elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>not_in_unknown_prj = [projects[prj] for prj in projects if prj != cls.GLOBAL_PROJECT]<EOL>not_in_unknown_sections = list(set([section for prj in not_in_unknown_prj<EOL>for section in list(prj.keys())]))<EOL>if backend_section not in not_in_unknown_sections:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>logger.debug("<STR_LIT>", backend_section, repos, raw)<EOL>repos = list(set(repos))<EOL>return repos<EOL>

return list with the repositories for a backend_section


projects = {}<EOL>projects['<STR_LIT>'] = {<EOL>"<STR_LIT>": ["<STR_LIT>"],<EOL>"<STR_LIT>": ["<STR_LIT>"]<EOL>}<EOL>projects = compose_title(projects, eclipse_projects)<EOL>projects = compose_projects_json(projects, eclipse_projects)<EOL>return projects<EOL>

Convert from eclipse projects format to grimoire projects json format


def __init__(self, tasks_cls, backend_section, stopper, config, timer=<NUM_LIT:0>):

super().__init__(name=backend_section)  <EOL>self.config = config<EOL>self.tasks_cls = tasks_cls  <EOL>self.tasks = []  <EOL>self.backend_section = backend_section<EOL>self.stopper = stopper  <EOL>self.timer = timer<EOL>self.thread_id = None<EOL>

:tasks_cls : tasks classes to be executed using the backend
:backend_section: perceval backend section name
:config: config object for the manager


version = None<EOL>es_url = self.conf['<STR_LIT>']['<STR_LIT:url>']<EOL>config_url = '<STR_LIT>'<EOL>url = urijoin(es_url, config_url)<EOL>version = None<EOL>try:<EOL><INDENT>res = self.grimoire_con.get(url)<EOL>res.raise_for_status()<EOL>version = res.json()['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>']<EOL>logger.debug("<STR_LIT>", version)<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT>return version<EOL>

Get the kibiter vesion.

        :param major: major Elasticsearch version


def create_dashboard(self, panel_file, data_sources=None, strict=True):

es_enrich = self.conf['<STR_LIT>']['<STR_LIT:url>']<EOL>kibana_url = self.conf['<STR_LIT>']['<STR_LIT>']<EOL>mboxes_sources = set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>if data_sources and any(x in data_sources for x in mboxes_sources):<EOL><INDENT>data_sources = list(data_sources)<EOL>data_sources.append('<STR_LIT>')<EOL><DEDENT>if data_sources and ('<STR_LIT>' in data_sources):<EOL><INDENT>data_sources = list(data_sources)<EOL>data_sources.append('<STR_LIT>')<EOL><DEDENT>if data_sources and '<STR_LIT>' in data_sources:<EOL><INDENT>data_sources = list(data_sources)<EOL>data_sources.append('<STR_LIT>')<EOL><DEDENT>if data_sources and '<STR_LIT>' in data_sources:<EOL><INDENT>data_sources = list(data_sources)<EOL>data_sources.append('<STR_LIT>')<EOL><DEDENT>if data_sources and '<STR_LIT>' in data_sources:<EOL><INDENT>data_sources = list(data_sources)<EOL>data_sources.append('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>import_dashboard(es_enrich, kibana_url, panel_file, data_sources=data_sources, strict=strict)<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.error("<STR_LIT>", panel_file)<EOL><DEDENT>except RuntimeError:<EOL><INDENT>logger.error("<STR_LIT>", panel_file)<EOL><DEDENT>

Upload a panel to Elasticsearch if it does not exist yet.

        If a list of data sources is specified, upload only those
        elements (visualizations, searches) that match that data source.

        :param panel_file: file name of panel (dashobard) to upload
        :param data_sources: list of data sources
        :param strict: only upload a dashboard if it is newer than the one already existing


if kibiter_major == "<STR_LIT>":<EOL><INDENT>resource = "<STR_LIT>"<EOL>data = {"<STR_LIT>": {"<STR_LIT:name>": self.project_name}}<EOL>mapping_resource = "<STR_LIT>"<EOL>mapping = {"<STR_LIT>": "<STR_LIT:true>"}<EOL>url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'], resource)<EOL>mapping_url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'],<EOL>mapping_resource)<EOL>logger.debug("<STR_LIT>")<EOL>res = self.grimoire_con.put(mapping_url, data=json.dumps(mapping),<EOL>headers=ES6_HEADER)<EOL>try:<EOL><INDENT>res.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>logger.error(res.json())<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>res = self.grimoire_con.post(url, data=json.dumps(data),<EOL>headers=ES6_HEADER)<EOL>try:<EOL><INDENT>res.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>logger.error(res.json())<EOL><DEDENT><DEDENT>

Upload to Kibiter the title for the dashboard.

        The title is shown on top of the dashboard menu, and is Usually
        the name of the project being dashboarded.
        This is done only for Kibiter 6.x.

        :param kibiter_major: major version of kibiter


def __create_dashboard_menu(self, dash_menu, kibiter_major):

logger.info("<STR_LIT>")<EOL>if kibiter_major == "<STR_LIT>":<EOL><INDENT>menu_resource = "<STR_LIT>"<EOL>mapping_resource = "<STR_LIT>"<EOL>mapping = {"<STR_LIT>": "<STR_LIT:true>"}<EOL>menu = {'<STR_LIT>': dash_menu}<EOL><DEDENT>else:<EOL><INDENT>menu_resource = "<STR_LIT>"<EOL>mapping_resource = "<STR_LIT>"<EOL>mapping = {"<STR_LIT>": "<STR_LIT:true>"}<EOL>menu = dash_menu<EOL><DEDENT>menu_url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'],<EOL>menu_resource)<EOL>mapping_url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'],<EOL>mapping_resource)<EOL>logger.debug("<STR_LIT>")<EOL>res = self.grimoire_con.put(mapping_url, data=json.dumps(mapping),<EOL>headers=ES6_HEADER)<EOL>try:<EOL><INDENT>res.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL><DEDENT>res = self.grimoire_con.post(menu_url, data=json.dumps(menu),<EOL>headers=ES6_HEADER)<EOL>try:<EOL><INDENT>res.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>logger.error(res.json())<EOL>raise<EOL><DEDENT>

Create the menu definition to access the panels in a dashboard.

        :param          menu: dashboard menu to upload
        :param kibiter_major: major version of kibiter


logger.info("<STR_LIT>")<EOL>if kibiter_major == "<STR_LIT>":<EOL><INDENT>metadashboard = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>metadashboard = "<STR_LIT>"<EOL><DEDENT>menu_url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'], metadashboard)<EOL>self.grimoire_con.delete(menu_url)<EOL>

Remove existing menu for dashboard, if any.

        Usually, we remove the menu before creating a new one.

        :param kibiter_major: major version of kibiter


menu_entries = []<EOL>for entry in self.panels_menu:<EOL><INDENT>if entry['<STR_LIT:source>'] not in self.data_sources:<EOL><INDENT>continue<EOL><DEDENT>parent_menu_item = {<EOL>'<STR_LIT:name>': entry['<STR_LIT:name>'],<EOL>'<STR_LIT:title>': entry['<STR_LIT:name>'],<EOL>'<STR_LIT:description>': "<STR_LIT>",<EOL>'<STR_LIT:type>': "<STR_LIT>",<EOL>'<STR_LIT>': []<EOL>}<EOL>for subentry in entry['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>dash_name = get_dashboard_name(subentry['<STR_LIT>'])<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>logging.error("<STR_LIT>", subentry['<STR_LIT>'])<EOL>continue<EOL><DEDENT>child_item = {<EOL>"<STR_LIT:name>": subentry['<STR_LIT:name>'],<EOL>"<STR_LIT:title>": subentry['<STR_LIT:name>'],<EOL>"<STR_LIT:description>": "<STR_LIT>",<EOL>"<STR_LIT:type>": "<STR_LIT>",<EOL>"<STR_LIT>": dash_name<EOL>}<EOL>parent_menu_item['<STR_LIT>'].append(child_item)<EOL><DEDENT>menu_entries.append(parent_menu_item)<EOL><DEDENT>return menu_entries<EOL>

Get the menu entries from the panel definition


<EOL>omenu = []<EOL>omenu.append(self.menu_panels_common['<STR_LIT>'])<EOL>ds_menu = self.__get_menu_entries(kibiter_major)<EOL>kafka_menu = None<EOL>community_menu = None<EOL>found_kafka = [pos for pos, menu in enumerate(ds_menu) if menu['<STR_LIT:name>'] == KAFKA_NAME]<EOL>if found_kafka:<EOL><INDENT>kafka_menu = ds_menu.pop(found_kafka[<NUM_LIT:0>])<EOL><DEDENT>found_community = [pos for pos, menu in enumerate(ds_menu) if menu['<STR_LIT:name>'] == COMMUNITY_NAME]<EOL>if found_community:<EOL><INDENT>community_menu = ds_menu.pop(found_community[<NUM_LIT:0>])<EOL><DEDENT>ds_menu.sort(key=operator.itemgetter('<STR_LIT:name>'))<EOL>omenu += ds_menu<EOL>if kafka_menu:<EOL><INDENT>omenu.append(kafka_menu)<EOL><DEDENT>if community_menu:<EOL><INDENT>omenu.append(community_menu)<EOL><DEDENT>omenu.append(self.menu_panels_common['<STR_LIT>'])<EOL>omenu.append(self.menu_panels_common['<STR_LIT>'])<EOL>logger.debug("<STR_LIT>", json.dumps(ds_menu, indent=<NUM_LIT:4>))<EOL>return omenu<EOL>


if '<STR_LIT>' not in self.conf[self.backend_section] or'<STR_LIT>' not in self.conf[self.backend_section]['<STR_LIT>']:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>aoc_index = self.conf['<STR_LIT>'].get('<STR_LIT>', GitEnrich.GIT_AOC_ENRICHED)<EOL>if not aoc_index:<EOL><INDENT>aoc_index = GitEnrich.GIT_AOC_ENRICHED<EOL><DEDENT>logger.debug("<STR_LIT>", aoc_index)<EOL>es = Elasticsearch([self.conf['<STR_LIT>']['<STR_LIT:url>']], timeout=<NUM_LIT:100>,<EOL>verify_certs=self._get_enrich_backend().elastic.requests.verify)<EOL>if not es.indices.exists(index=aoc_index):<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>aoc_backend = GitEnrich(self.db_sh, None, cfg['<STR_LIT>']['<STR_LIT>'],<EOL>self.db_user, self.db_password, self.db_host)<EOL>aoc_backend.mapping = None<EOL>aoc_backend.roles = ['<STR_LIT>']<EOL>elastic_enrich = get_elastic(self.conf['<STR_LIT>']['<STR_LIT:url>'],<EOL>aoc_index, clean=False, backend=aoc_backend)<EOL>aoc_backend.set_elastic(elastic_enrich)<EOL>self.__autorefresh(aoc_backend, studies=True)<EOL>

Execute autorefresh for areas of code study if configured


cfg = self.config.get_conf()<EOL>if '<STR_LIT>' not in cfg[self.backend_section] or notcfg[self.backend_section]['<STR_LIT>']:<EOL><INDENT>logger.debug('<STR_LIT>' % self.backend_section)<EOL>return<EOL><DEDENT>studies = [study for study in cfg[self.backend_section]['<STR_LIT>'] if study.strip() != "<STR_LIT>"]<EOL>if not studies:<EOL><INDENT>logger.debug('<STR_LIT>' % self.backend_section)<EOL>return<EOL><DEDENT>logger.debug("<STR_LIT>" % (self.backend_section, studies))<EOL>time.sleep(<NUM_LIT:2>)  <EOL>enrich_backend = self._get_enrich_backend()<EOL>ocean_backend = self._get_ocean_backend(enrich_backend)<EOL>active_studies = []<EOL>all_studies = enrich_backend.studies<EOL>all_studies_names = [study.__name__ for study in enrich_backend.studies]<EOL>logger.debug("<STR_LIT>", self.backend_section, all_studies_names)<EOL>logger.debug("<STR_LIT>", studies)<EOL>cfg_studies_types = [study.split("<STR_LIT::>")[<NUM_LIT:0>] for study in studies]<EOL>if not set(cfg_studies_types).issubset(set(all_studies_names)):<EOL><INDENT>logger.error('<STR_LIT>', self.backend_section, studies)<EOL>raise RuntimeError('<STR_LIT>', self.backend_section, studies)<EOL><DEDENT>for study in enrich_backend.studies:<EOL><INDENT>if study.__name__ in cfg_studies_types:<EOL><INDENT>active_studies.append(study)<EOL><DEDENT><DEDENT>enrich_backend.studies = active_studies<EOL>print("<STR_LIT>" % (self.backend_section,<EOL>[study for study in studies]))<EOL>studies_args = self.__load_studies()<EOL>do_studies(ocean_backend, enrich_backend, studies_args, retention_time=retention_time)<EOL>enrich_backend.studies = all_studies<EOL>

Execute the studies configured for the current backend


enrich_es = self.conf['<STR_LIT>']['<STR_LIT:url>']<EOL>sortinghat_db = self.db<EOL>current_data_source = self.get_backend(self.backend_section)<EOL>active_data_sources = self.config.get_active_data_sources()<EOL>if retention_time is None:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>if retention_time <= <NUM_LIT:0>:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>retain_identities(retention_time, enrich_es, sortinghat_db, current_data_source, active_data_sources)<EOL>

Retain the identities in SortingHat based on the `retention_time`
        value declared in the setup.cfg.

        :param retention_time: maximum number of minutes wrt the current date to retain the SortingHat data


def micro_mordred(cfg_path, backend_sections, raw, arthur, identities, enrich, panels):

config = Config(cfg_path)<EOL>if raw:<EOL><INDENT>for backend in backend_sections:<EOL><INDENT>get_raw(config, backend, arthur)<EOL><DEDENT><DEDENT>if identities:<EOL><INDENT>get_identities(config)<EOL><DEDENT>if enrich:<EOL><INDENT>for backend in backend_sections:<EOL><INDENT>get_enrich(config, backend)<EOL><DEDENT><DEDENT>if panels:<EOL><INDENT>get_panels(config)<EOL><DEDENT>

Execute the raw and/or the enrich phases of a given backend section defined in a Mordred configuration file.

    :param cfg_path: the path of a Mordred configuration file
    :param backend_sections: the backend sections where the raw and/or enrich phases will be executed
    :param raw: if true, it activates the collection of raw data
    :param arthur: if true, it enables Arthur to collect the raw data
    :param identities: if true, it activates the identities merge in SortingHat
    :param enrich: if true, it activates the collection of enrich data
    :param panels: if true, it activates the upload of panels


def get_raw(config, backend_section, arthur):

if arthur:<EOL><INDENT>task = TaskRawDataArthurCollection(config, backend_section=backend_section)<EOL><DEDENT>else:<EOL><INDENT>task = TaskRawDataCollection(config, backend_section=backend_section)<EOL><DEDENT>TaskProjects(config).execute()<EOL>try:<EOL><INDENT>task.execute()<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>except Exception as e:<EOL><INDENT>logging.error(str(e))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>

Execute the raw phase for a given backend section, optionally using Arthur

    :param config: a Mordred config object
    :param backend_section: the backend section where the raw phase is executed
    :param arthur: if true, it enables Arthur to collect the raw data


TaskProjects(config).execute()<EOL>task = TaskIdentitiesMerge(config)<EOL>task.execute()<EOL>logging.info("<STR_LIT>")<EOL>

Execute the merge identities phase

    :param config: a Mordred config object


TaskProjects(config).execute()<EOL>task = TaskEnrich(config, backend_section=backend_section)<EOL>try:<EOL><INDENT>task.execute()<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>except Exception as e:<EOL><INDENT>logging.error(str(e))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>

Execute the enrich phase for a given backend section

    :param config: a Mordred config object
    :param backend_section: the backend section where the enrich phase is executed


task = TaskPanels(config)<EOL>task.execute()<EOL>task = TaskPanelsMenu(config)<EOL>task.execute()<EOL>logging.info("<STR_LIT>")<EOL>

Execute the panels phase

    :param config: a Mordred config object


parser = argparse.ArgumentParser(add_help=False)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>help=argparse.SUPPRESS)<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", dest='<STR_LIT>', default=[],<EOL>nargs='<STR_LIT:*>', help="<STR_LIT>")<EOL>if len(sys.argv) == <NUM_LIT:1>:<EOL><INDENT>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return parser<EOL>


parser = get_params_parser()<EOL>args = parser.parse_args()<EOL>if not args.raw and not args.enrich and not args.identities and not args.panels:<EOL><INDENT>print("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return args<EOL>


return random.randint(<NUM_LIT:0>, <NUM_LIT:255>), random.randint(<NUM_LIT:0>, <NUM_LIT:255>), random.randint(<NUM_LIT:0>, <NUM_LIT:255>)<EOL>

Generate a uniformly random RGB value.

:return: A tuple of three integers with values between 0 and 255 inclusive


return rgb_to_hex(random_rgb())<EOL>

Generate a uniformly random HEX value.

:return: A string representing a random HEX value between 000000 and FFFFFF inclusive


return rgb_to_web(random_rgb())<EOL>

Generate a uniformly random WEB value.

:return:


r, g, b = seed<EOL>results = []<EOL>for _ in range(amount):<EOL><INDENT>base_val = ((r + g + b) / <NUM_LIT:3>) + <NUM_LIT:1>  <EOL>new_val = base_val + (random.random() * rgb_max_val / <NUM_LIT:5>)  <EOL>ratio = new_val / base_val<EOL>results.append((min(int(r*ratio), rgb_max_val), min(int(g*ratio), rgb_max_val), min(int(b*ratio), rgb_max_val)))<EOL><DEDENT>return results[<NUM_LIT:0>] if len(results) > <NUM_LIT:1> else results<EOL>

Given a seed color, generate a specified number of random colors (1 color by default) determined by a randomized
offset from the seed.

:param seed:
:param amount:
:return:


return rgb_to_hex(offset_random_rgb(seed, amount))<EOL>


return rgb_to_web(offset_random_rgb(seed, amount))<EOL>


def color_run(start_color, end_color, step_count, inclusive=True, to_color=True):

if isinstance(start_color, Color):<EOL><INDENT>start_color = start_color.rgb<EOL><DEDENT>if isinstance(end_color, Color):<EOL><INDENT>end_color = end_color.rgb<EOL><DEDENT>step = tuple((end_color[i] - start_color[i])/step_count for i in range(<NUM_LIT:3>))<EOL>add = lambda x, y: tuple(sum(z) for z in zip(x, y))<EOL>mult = lambda x, y: tuple(y * z for z in x)<EOL>run = [add(start_color, mult(step, i)) for i in range(<NUM_LIT:1>, step_count)]<EOL>if inclusive:<EOL><INDENT>run = [start_color] + run + [end_color]<EOL><DEDENT>return run if not to_color else [Color(c) for c in run]<EOL>

Given a start color, end color, and a number of steps, returns a list of colors which represent a 'scale' between
the start and end color.

:param start_color: The color starting the run
:param end_color: The color ending the run
:param step_count: The number of colors to have between the start and end color
:param inclusive: Flag determining whether to include start and end values in run (default True)
:param to_color: Flag indicating return values should be Color objects (default True)
:return: List of colors between the start and end color
:rtype: list


def text_color(background, dark_color=rgb_min, light_color=rgb_max):

max_y = rgb_to_yiq(rgb_max)[<NUM_LIT:0>]<EOL>return light_color if rgb_to_yiq(background)[<NUM_LIT:0>] <= max_y / <NUM_LIT:2> else dark_color<EOL>

Given a background color in the form of an RGB 3-tuple, returns the color the text should be (defaulting to white
and black) for best readability. The light (white) and dark (black) defaults can be overridden to return preferred
values.

:param background:
:param dark_color:
:param light_color:
:return:


size = len(_hex.strip('<STR_LIT:#>'))<EOL>if size == <NUM_LIT:3>:<EOL><INDENT>return _hex<EOL><DEDENT>elif size == <NUM_LIT:6>:<EOL><INDENT>if _hex[<NUM_LIT:1>] == _hex[<NUM_LIT:2>] and _hex[<NUM_LIT:3>] == _hex[<NUM_LIT:4>] and _hex[<NUM_LIT:5>] == _hex[<NUM_LIT:6>]:<EOL><INDENT>return _hex[<NUM_LIT:0>::<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>return _hex<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ColorException('<STR_LIT>'.format(size))<EOL><DEDENT>

Given a HEX value, tries to reduce it from a 6 character hex (e.g. #ffffff) to a 3 character hex (e.g. #fff).
If the HEX value is unable to be minified, returns the 6 character HEX representation.

:param _hex:
:return:


def __init__(self, color=None, **kwargs):

self.equality_fn = RGB_eq<EOL>self.arithmetic = ArithmeticModel.LIGHT<EOL>if isinstance(color, Color):<EOL><INDENT>self._color = color._color<EOL><DEDENT>else:<EOL><INDENT>self._color = color if color else rgb_min<EOL><DEDENT>for k, v in kwargs.items():<EOL><INDENT>setattr(self, k, v)<EOL><DEDENT>


if isinstance(other, Color):<EOL><INDENT>return self.equality_fn(self, other)<EOL><DEDENT>return False<EOL>


return not self.__eq__(other)<EOL>


if isinstance(other, Color):<EOL><INDENT>r1, g1, b1 = self.rgb<EOL>r2, g2, b2 = other.rgb<EOL><DEDENT>elif isinstance(other, tuple) and len(other) == <NUM_LIT:3>:<EOL><INDENT>r1, g1, b1 = self.rgb<EOL>r2, g2, b2 = other<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>".format(type(self), type(other)))<EOL><DEDENT>if self.arithmetic is ArithmeticModel.LIGHT:<EOL><INDENT>return Color((min(r1 + r2, rgb_max_val), min(g1 + g2, rgb_max_val), min(b1 + b2, rgb_max_val)))<EOL><DEDENT>else:<EOL><INDENT>return Color(((r1 + r2 // <NUM_LIT:2>), (g1 + g2 // <NUM_LIT:2>), (b1 + b2 // <NUM_LIT:2>)))<EOL><DEDENT>


if isinstance(other, Color):<EOL><INDENT>r1, g1, b1 = self.rgb<EOL>r2, g2, b2 = other.rgb<EOL><DEDENT>elif isinstance(other, tuple) and len(other) == <NUM_LIT:3>:<EOL><INDENT>r1, g1, b1 = self.rgb<EOL>r2, g2, b2 = other<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>".format(type(self), type(other)))<EOL><DEDENT>return Color((max(r1 - r2, rgb_min_val), max(g1 - g2, rgb_min_val), max(b1 - b2, rgb_min_val)))<EOL>


return iter(self._color)<EOL>


return "<STR_LIT:{}>".format(self._color)<EOL>


return "<STR_LIT>".format(self._color)<EOL>


return self._color[<NUM_LIT:0>]<EOL>

The red component of the RGB color representation.


return self._color[<NUM_LIT:1>]<EOL>

The green component of the RGB color representation.


return self._color[<NUM_LIT:2>]<EOL>

The blue component of the RGB color representation.


return self._color<EOL>


return rgb_to_hex(self.rgb)<EOL>

A 6-char HEX representation of the color, with a prepended octothorpe.


return minify_hex(self.hex)<EOL>

The same as Color.hex, however, HEX values that can be minified to 3-char are returned as such.


return rgb_to_web(self.rgb)<EOL>


return rgb_to_yiq(self.rgb)<EOL>


return rgb_to_hsv(self.rgb)<EOL>


r, g, b = rgb<EOL>return "<STR_LIT>".format(hex(int(r))[<NUM_LIT:2>:].zfill(<NUM_LIT:2>), hex(int(g))[<NUM_LIT:2>:].zfill(<NUM_LIT:2>), hex(int(b))[<NUM_LIT:2>:].zfill(<NUM_LIT:2>))<EOL>

Convert an RGB color representation to a HEX color representation.

(r, g, b) :: r -> [0, 255]
             g -> [0, 255]
             b -> [0, 255]

:param rgb: A tuple of three numeric values corresponding to the red, green, and blue value.
:return: HEX representation of the input RGB value.
:rtype: str


try:<EOL><INDENT>return web_colors[rgb]<EOL><DEDENT>except KeyError:<EOL><INDENT>return rgb_to_hex(rgb)<EOL><DEDENT>

Convert an RGB color representation to a WEB color representation.

(r, g, b) :: r -> [0, 255]
             g -> [0, 255]
             b -> [0, 255]

:param rgb: A tuple of three numeric values corresponding to the red, green, and blue value.
:return: WEB representation of the input RGB value.
:rtype: str


r, g, b = rgb[<NUM_LIT:0>] / <NUM_LIT:255>, rgb[<NUM_LIT:1>] / <NUM_LIT:255>, rgb[<NUM_LIT:2>] / <NUM_LIT:255><EOL>y = (<NUM_LIT> * r) + (<NUM_LIT> * g) + (<NUM_LIT> * b)<EOL>i = (<NUM_LIT> * r) - (<NUM_LIT> * g) - (<NUM_LIT> * b)<EOL>q = (<NUM_LIT> * r) - (<NUM_LIT> * g) + (<NUM_LIT> * b)<EOL>return round(y, <NUM_LIT:3>), round(i, <NUM_LIT:3>), round(q, <NUM_LIT:3>)<EOL>

Convert an RGB color representation to a YIQ color representation.

(r, g, b) :: r -> [0, 255]
             g -> [0, 255]
             b -> [0, 255]

:param rgb: A tuple of three numeric values corresponding to the red, green, and blue value.
:return: YIQ representation of the input RGB value.
:rtype: tuple


r, g, b = rgb[<NUM_LIT:0>] / <NUM_LIT:255>, rgb[<NUM_LIT:1>] / <NUM_LIT:255>, rgb[<NUM_LIT:2>] / <NUM_LIT:255><EOL>_min = min(r, g, b)<EOL>_max = max(r, g, b)<EOL>v = _max<EOL>delta = _max - _min<EOL>if _max == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0>, <NUM_LIT:0>, v<EOL><DEDENT>s = delta / _max<EOL>if delta == <NUM_LIT:0>:<EOL><INDENT>delta = <NUM_LIT:1><EOL><DEDENT>if r == _max:<EOL><INDENT>h = <NUM_LIT> * (((g - b) / delta) % <NUM_LIT:6>)<EOL><DEDENT>elif g == _max:<EOL><INDENT>h = <NUM_LIT> * (((b - r) / delta) + <NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>h = <NUM_LIT> * (((r - g) / delta) + <NUM_LIT:4>)<EOL><DEDENT>return round(h, <NUM_LIT:3>), round(s, <NUM_LIT:3>), round(v, <NUM_LIT:3>)<EOL>

Convert an RGB color representation to an HSV color representation.

(r, g, b) :: r -> [0, 255]
             g -> [0, 255]
             b -> [0, 255]

:param rgb: A tuple of three numeric values corresponding to the red, green, and blue value.
:return: HSV representation of the input RGB value.
:rtype: tuple


_hex = _hex.strip('<STR_LIT:#>')<EOL>n = len(_hex) // <NUM_LIT:3><EOL>if len(_hex) == <NUM_LIT:3>:<EOL><INDENT>r = int(_hex[:n] * <NUM_LIT:2>, <NUM_LIT:16>)<EOL>g = int(_hex[n:<NUM_LIT:2> * n] * <NUM_LIT:2>, <NUM_LIT:16>)<EOL>b = int(_hex[<NUM_LIT:2> * n:<NUM_LIT:3> * n] * <NUM_LIT:2>, <NUM_LIT:16>)<EOL><DEDENT>else:<EOL><INDENT>r = int(_hex[:n], <NUM_LIT:16>)<EOL>g = int(_hex[n:<NUM_LIT:2> * n], <NUM_LIT:16>)<EOL>b = int(_hex[<NUM_LIT:2> * n:<NUM_LIT:3> * n], <NUM_LIT:16>)<EOL><DEDENT>return r, g, b<EOL>

Convert a HEX color representation to an RGB color representation.

hex :: hex -> [000000, FFFFFF]

:param _hex: The 3- or 6-char hexadecimal string representing the color value.
:return: RGB representation of the input HEX value.
:rtype: tuple


try:<EOL><INDENT>return web_colors[hex_to_rgb(_hex)]<EOL><DEDENT>except KeyError:<EOL><INDENT>return _hex<EOL><DEDENT>

Convert a HEX color representation to a WEB color representation.

hex :: hex -> [000000, FFFFFF]

:param _hex: The 3- or 6-char hexadecimal string representing the color value.
:return: WEB representation of the input HEX value.
:rtype: str


return rgb_to_yiq(hex_to_rgb(_hex))<EOL>

Convert a HEX color representation to a YIQ color representation.

hex :: hex -> [000000, FFFFFF]

:param _hex: The 3- or 6-char hexadecimal string representing the color value.
:return: YIQ representation of the input HEX value.
:rtype: tuple


return rgb_to_hsv(hex_to_rgb(_hex))<EOL>

Convert a HEX color representation to an HSV color representation.

hex :: hex -> [000000, FFFFFF]

:param _hex: The 3- or 6-char hexadecimal string representing the color value.
:return: HSV representation of the input HEX value.
:rtype: tuple


try:<EOL><INDENT>return web_colors[web.lower()]<EOL><DEDENT>except KeyError:<EOL><INDENT>return hex_to_rgb(web)<EOL><DEDENT>

Convert a WEB color representation to an RGB color representation.

web :: web -> [000000, FFFFFF]
            | in static.web_colors

:param web: The WEB string representation of a color.
:return: RGB representation of the input WEB value.
:rtype: tuple


try:<EOL><INDENT>return rgb_to_hex(web_colors[web])<EOL><DEDENT>except KeyError:<EOL><INDENT>return web<EOL><DEDENT>

Convert a WEB color representation to a HEX color representation.

web :: web -> [000000, FFFFFF]
            | in static.web_colors

:param web: The WEB string representation of a color.
:return: HEX representation of the input WEB value.
:rtype: str


return rgb_to_yiq(web_to_rgb(web))<EOL>

Convert a WEB color representation to a YIQ color representation.

web :: web -> [000000, FFFFFF]
            | in static.web_colors

:param web: The WEB string representation of a color.
:return: YIQ representation of the input WEB value.
:rtype: tuple


return rgb_to_hsv(web_to_rgb(web))<EOL>

Convert a WEB color representation to an HSV color representation.

web :: web -> [000000, FFFFFF]
            | in static.web_colors

:param web: The WEB string representation of a color.
:return: HSV representation of the input WEB value.
:rtype: tuple


y, i, q = yiq<EOL>r = y + (<NUM_LIT> * i) + (<NUM_LIT> * q)<EOL>g = y - (<NUM_LIT> * i) - (<NUM_LIT> * q)<EOL>b = y - (<NUM_LIT> * i) + (<NUM_LIT> * q)<EOL>r = <NUM_LIT:1> if r > <NUM_LIT:1> else max(<NUM_LIT:0>, r)<EOL>g = <NUM_LIT:1> if g > <NUM_LIT:1> else max(<NUM_LIT:0>, g)<EOL>b = <NUM_LIT:1> if b > <NUM_LIT:1> else max(<NUM_LIT:0>, b)<EOL>return round(r * <NUM_LIT:255>, <NUM_LIT:3>), round(g * <NUM_LIT:255>, <NUM_LIT:3>), round(b * <NUM_LIT:255>, <NUM_LIT:3>)<EOL>

Convert a YIQ color representation to an RGB color representation.

(y, i, q) :: y -> [0, 1]
             i -> [-0.5957, 0.5957]
             q -> [-0.5226, 0.5226]

:param yiq: A tuple of three numeric values corresponding to the luma and chrominance.
:return: RGB representation of the input YIQ value.
:rtype: tuple


return rgb_to_hex(yiq_to_rgb(yiq))<EOL>

Convert a YIQ color representation to a HEX color representation.

(y, i, q) :: y -> [0, 1]
             i -> [-0.5957, 0.5957]
             q -> [-0.5226, 0.5226]

:param yiq: A tuple of three numeric values corresponding to the luma and chrominance.
:return: HEX representation of the input YIQ value.
:rtype: str


return rgb_to_web(yiq_to_rgb(yiq))<EOL>

Convert a YIQ color representation to a WEB color representation.

(y, i, q) :: y -> [0, 1]
             i -> [-0.5957, 0.5957]
             q -> [-0.5226, 0.5226]

:param yiq: A tuple of three numeric values corresponding to the luma and chrominance.
:return: WEB representation of the input YIQ value.
:rtype: str


return rgb_to_hsv(yiq_to_rgb(yiq))<EOL>

Convert a YIQ color representation to an HSV color representation.

(y, i, q) :: y -> [0, 1]
             i -> [-0.5957, 0.5957]
             q -> [-0.5226, 0.5226]

:param yiq: A tuple of three numeric values corresponding to the luma and chrominance.
:return: HSV representation of the input YIQ value.
:rtype: tuple


h, s, v = hsv<EOL>c = v * s<EOL>h /= <NUM_LIT><EOL>x = c * (<NUM_LIT:1> - abs((h % <NUM_LIT:2>) - <NUM_LIT:1>))<EOL>m = v - c<EOL>if h < <NUM_LIT:1>:<EOL><INDENT>res = (c, x, <NUM_LIT:0>)<EOL><DEDENT>elif h < <NUM_LIT:2>:<EOL><INDENT>res = (x, c, <NUM_LIT:0>)<EOL><DEDENT>elif h < <NUM_LIT:3>:<EOL><INDENT>res = (<NUM_LIT:0>, c, x)<EOL><DEDENT>elif h < <NUM_LIT:4>:<EOL><INDENT>res = (<NUM_LIT:0>, x, c)<EOL><DEDENT>elif h < <NUM_LIT:5>:<EOL><INDENT>res = (x, <NUM_LIT:0>, c)<EOL><DEDENT>elif h < <NUM_LIT:6>:<EOL><INDENT>res = (c, <NUM_LIT:0>, x)<EOL><DEDENT>else:<EOL><INDENT>raise ColorException("<STR_LIT>")<EOL><DEDENT>r, g, b = res<EOL>return round((r + m)*<NUM_LIT:255>, <NUM_LIT:3>), round((g + m)*<NUM_LIT:255>, <NUM_LIT:3>), round((b + m)*<NUM_LIT:255>, <NUM_LIT:3>)<EOL>

Convert an HSV color representation to an RGB color representation.

(h, s, v) :: h -> [0, 360)
             s -> [0, 1]
             v -> [0, 1]

:param hsv: A tuple of three numeric values corresponding to the hue, saturation, and value.
:return: RGB representation of the input HSV value.
:rtype: tuple


return rgb_to_hex(hsv_to_rgb(hsv))<EOL>

Convert an HSV color representation to a HEX color representation.

(h, s, v) :: h -> [0, 360)
             s -> [0, 1]
             v -> [0, 1]

:param hsv: A tuple of three numeric values corresponding to the hue, saturation, and value.
:return: HEX representation of the input HSV value.
:rtype: str


return rgb_to_web(hsv_to_rgb(hsv))<EOL>

Convert an HSV color representation to a WEB color representation.

(h, s, v) :: h -> [0, 360)
             s -> [0, 1]
             v -> [0, 1]

:param hsv: A tuple of three numeric values corresponding to the hue, saturation, and value.
:return: WEB representation of the input HSV value.
:rtype: str


return rgb_to_yiq(hsv_to_rgb(hsv))<EOL>

Convert an HSV color representation to a YIQ color representation.

(h, s, v) :: h -> [0, 360)
             s -> [0, 1]
             v -> [0, 1]

:param hsv: A tuple of three numeric values corresponding to the hue, saturation, and value.
:return: YIQ representation of the input HSV value.
:rtype: tuple


def __init__(self, chain_order=<NUM_LIT:2>, vk_object=None, *file_paths):

if chain_order < <NUM_LIT:1>:<EOL><INDENT>raise MarkovTextExcept(<EOL>"<STR_LIT>".format(chain_order)<EOL>)<EOL><DEDENT>self.chain_order = chain_order<EOL>self.tokens_array = ()<EOL>self.base_dict = {}<EOL>self.start_arrays = ()<EOL>self.vk_object = vk_object<EOL>self.vocabulars = {}<EOL>self.temp_folder = abspath(<EOL>os_join(expanduser("<STR_LIT>"), "<STR_LIT>")<EOL>)<EOL>if not isdir(self.temp_folder):<EOL><INDENT>makedirs(self.temp_folder)<EOL><DEDENT>for _path in frozenset(filter(isfile, map(abspath, file_paths))):<EOL><INDENT>self.update(_path)<EOL><DEDENT>

:chain_order: Количество звеньев цепи, для принятия решения о следующем.
:vk_object: Объект класса Владя-бота, для интеграции. Не обязателен.
:file_paths: Пути к текстовым файлам, для обучения модели.


if self.is_rus_word(token):<EOL><INDENT>return True<EOL><DEDENT>elif self.ONLY_MARKS.search(token):<EOL><INDENT>return True<EOL><DEDENT>elif self.END_TOKENS.search(token):<EOL><INDENT>return True<EOL><DEDENT>elif token in "<STR_LIT>":<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

Подходит ли токен, для генерации текста.
Допускаются русские слова, знаки препинания и символы начала и конца.


def get_optimal_variant(self, variants, start_words, **kwargs):

if not start_words:<EOL><INDENT>return (choice(variants), {})<EOL><DEDENT>_variants = []<EOL>_weights = []<EOL>for tok in frozenset(variants):<EOL><INDENT>if not self.token_is_correct(tok):<EOL><INDENT>continue<EOL><DEDENT>weight = variants.count(tok)<EOL>for word in start_words:<EOL><INDENT>for token in self.ONLY_WORDS.finditer(word.strip().lower()):<EOL><INDENT>if token.group() == tok:<EOL><INDENT>weight <<= <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>_variants.append(tok)<EOL>_weights.append(weight)<EOL><DEDENT>if not _variants:<EOL><INDENT>return (choice(variants), {})<EOL><DEDENT>return (choices(_variants, weights=_weights, k=<NUM_LIT:1>)[<NUM_LIT:0>], {})<EOL>

Возвращает оптимальный вариант, из выборки.


def start_generation(self, *start_words, **kwargs):

out_text = "<STR_LIT>"<EOL>_need_capialize = True<EOL>for token in self._get_generate_tokens(*start_words, **kwargs):<EOL><INDENT>if token in "<STR_LIT>":<EOL><INDENT>_need_capialize = True<EOL>continue<EOL><DEDENT>if self.ONLY_WORDS.search(token):<EOL><INDENT>out_text += "<STR_LIT:U+0020>"<EOL><DEDENT>if _need_capialize:<EOL><INDENT>_need_capialize = False<EOL>token = token.title()<EOL><DEDENT>out_text += token<EOL><DEDENT>return out_text.strip()<EOL>

Генерирует предложение.
:start_words: Попытаться начать предложение с этих слов.


def get_start_array(self, *start_words, **kwargs):

if not self.start_arrays:<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>")<EOL><DEDENT>if not start_words:<EOL><INDENT>return choice(self.start_arrays)<EOL><DEDENT>_variants = []<EOL>_weights = []<EOL>for tokens in self.start_arrays:<EOL><INDENT>weight = <NUM_LIT><EOL>for word in start_words:<EOL><INDENT>word = word.strip().lower()<EOL>for token in self.ONLY_WORDS.finditer(word):<EOL><INDENT>if token.group() in tokens:<EOL><INDENT>weight <<= <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>if weight > <NUM_LIT>:<EOL><INDENT>_variants.append(tokens)<EOL>_weights.append(weight)<EOL><DEDENT><DEDENT>if not _variants:<EOL><INDENT>return choice(self.start_arrays)<EOL><DEDENT>return choices(_variants, weights=_weights, k=<NUM_LIT:1>)[<NUM_LIT:0>]<EOL>

Генерирует начало предложения.
:start_words: Попытаться начать предложение с этих слов.


self.base_dict = {}<EOL>_start_arrays = []<EOL>for tokens, word in self.chain_generator():<EOL><INDENT>self.base_dict.setdefault(tokens, []).append(word)<EOL>if tokens[<NUM_LIT:0>] == "<STR_LIT>":  <EOL><INDENT>_start_arrays.append(tokens)<EOL><DEDENT><DEDENT>self.start_arrays = tuple(<EOL>frozenset(self.get_corrected_arrays(_start_arrays))<EOL>)<EOL>

Метод создаёт базовый словарь, на основе массива токенов.
Вызывается из метода обновления.


n_chain = self.chain_order<EOL>if n_chain < <NUM_LIT:1>:<EOL><INDENT>raise MarkovTextExcept(<EOL>"<STR_LIT>".format(n_chain)<EOL>)<EOL><DEDENT>n_chain += <NUM_LIT:1>  <EOL>changing_array = deque(maxlen=n_chain)<EOL>for token in self.tokens_array:<EOL><INDENT>changing_array.append(token)<EOL>if len(changing_array) < n_chain:<EOL><INDENT>continue  <EOL><DEDENT>yield (tuple(changing_array)[:-<NUM_LIT:1>], changing_array[-<NUM_LIT:1>])<EOL><DEDENT>

Возвращает генератор, формата:
    (("токен", ...), "вариант")
    Где количество токенов определяет переменная объекта chain_order.


def set_vocabulary(self, peer_id, from_dialogue=None, update=False):

self.tokens_array = self.get_vocabulary(<EOL>peer_id,<EOL>from_dialogue,<EOL>update<EOL>)<EOL>self.create_base()<EOL>

Получает вокабулар из функции get_vocabulary и делает его активным.


name = name or "<STR_LIT>"<EOL>backup_dump_file = os_join(<EOL>self.temp_folder,<EOL>"<STR_LIT>".format(name)<EOL>)<EOL>dump_file = os_join(<EOL>self.temp_folder,<EOL>"<STR_LIT>".format(name)<EOL>)<EOL>with open(backup_dump_file, "<STR_LIT:w>", encoding="<STR_LIT:utf-8>") as js_file:<EOL><INDENT>json.dump(self.tokens_array, js_file, ensure_ascii=False)<EOL><DEDENT>copy2(backup_dump_file, dump_file)<EOL>remove(backup_dump_file)<EOL>

Сохраняет текущую базу на жёсткий диск.
:name: Имя файла, без расширения.


name = name or "<STR_LIT>"<EOL>dump_file = os_join(<EOL>self.temp_folder,<EOL>"<STR_LIT>".format(name)<EOL>)<EOL>if not isfile(dump_file):<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>".format(dump_file))<EOL><DEDENT>with open(dump_file, "<STR_LIT:rb>") as js_file:<EOL><INDENT>self.tokens_array = tuple(json.load(js_file))<EOL><DEDENT>self.create_base()<EOL>

Загружает базу с жёсткого диска.
Текущая база заменяется.
:name: Имя файла, без расширения.


def get_vocabulary(self, target, user=None, update=False):

if not self.vk_object:<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>")<EOL><DEDENT>json_name = "<STR_LIT>".format(<EOL>target.__class__.__name__,<EOL>target.id,<EOL>user.id<EOL>)<EOL>json_file = os_join(self.temp_folder, "<STR_LIT>".format(json_name))<EOL>if not update:<EOL><INDENT>result = self.vocabulars.get(json_name, None)<EOL>if result:<EOL><INDENT>return result<EOL><DEDENT>elif isfile(json_file):<EOL><INDENT>with open(json_file, "<STR_LIT:rb>") as js_file:<EOL><INDENT>result = self.vocabulars[json_name] = tuple(<EOL>json.load(js_file)<EOL>)<EOL><DEDENT>return result<EOL><DEDENT><DEDENT>_tokens_array = tuple(self.__parse_from_vk_dialogue(target, user))<EOL>backup_file = "<STR_LIT>".format(splitext(json_file)[<NUM_LIT:0>])<EOL>with open(backup_file, "<STR_LIT:w>", encoding="<STR_LIT:utf-8>") as js_file:<EOL><INDENT>json.dump(_tokens_array, js_file, ensure_ascii=False)<EOL><DEDENT>copy2(backup_file, json_file)<EOL>remove(backup_file)<EOL>self.vocabulars[json_name] = _tokens_array<EOL>return _tokens_array<EOL>

Возвращает запас слов, на основе переписок ВК.
Для имитации речи конкретного человека.
Работает только с импортом объекта "Владя-бота".

:target:
    Объект собседника. Источник переписки.
:user:
    Объект юзера, чью речь имитируем.
    Если None, то вся переписка.
:update:
    Не использовать бэкап. Обновить форсированно.


func = (self._parse_from_file if fromfile else self._parse_from_text)<EOL>new_data = tuple(func(data))<EOL>if new_data:<EOL><INDENT>self.tokens_array += new_data<EOL>self.create_base()<EOL><DEDENT>

Принимает текст, или путь к файлу и обновляет существующую базу.


if not isinstance(text, str):<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>")<EOL><DEDENT>text = text.strip().lower()<EOL>need_start_token = True<EOL>token = "<STR_LIT:$>"  <EOL>for token in self.WORD_OR_MARKS.finditer(text):<EOL><INDENT>token = token.group()<EOL>if need_start_token:<EOL><INDENT>need_start_token = False<EOL>yield "<STR_LIT>"<EOL><DEDENT>yield token<EOL>if self.END_TOKENS.search(token):<EOL><INDENT>need_start_token = True<EOL>yield "<STR_LIT:$>"<EOL><DEDENT><DEDENT>if token != "<STR_LIT:$>":<EOL><INDENT>yield "<STR_LIT:$>"<EOL><DEDENT>

Возвращает генератор токенов, из текста.


file_path = abspath(file_path)<EOL>if not isfile(file_path):<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>")<EOL><DEDENT>with open(file_path, "<STR_LIT:rb>") as txt_file:<EOL><INDENT>for line in txt_file:<EOL><INDENT>text = line.decode("<STR_LIT:utf-8>", "<STR_LIT:ignore>").strip()<EOL>if not text:<EOL><INDENT>continue<EOL><DEDENT>yield from self._parse_from_text(text)<EOL><DEDENT><DEDENT>

см. описание _parse_from_text.
Только на вход подаётся не текст, а путь к файлу.


x = xxh32()<EOL>for i in lines:<EOL><INDENT>x.update(i.encode())<EOL><DEDENT>return x.digest()<EOL>

Creates a unique binary id for the given lines
Args:
    lines (list<str>): List of strings that should be collectively hashed
Returns:
    bytearray: Binary hash


tokens = []<EOL>class Vars:<EOL><INDENT>start_pos = -<NUM_LIT:1><EOL>last_type = '<STR_LIT:o>'<EOL><DEDENT>def update(c, i):<EOL><INDENT>if c.isalpha() or c in '<STR_LIT>':<EOL><INDENT>t = '<STR_LIT:a>'<EOL><DEDENT>elif c.isdigit() or c == '<STR_LIT:#>':<EOL><INDENT>t = '<STR_LIT:n>'<EOL><DEDENT>elif c.isspace():<EOL><INDENT>t = '<STR_LIT:s>'<EOL><DEDENT>else:<EOL><INDENT>t = '<STR_LIT:o>'<EOL><DEDENT>if t != Vars.last_type or t == '<STR_LIT:o>':<EOL><INDENT>if Vars.start_pos >= <NUM_LIT:0>:<EOL><INDENT>token = sentence[Vars.start_pos:i].lower()<EOL>if token not in '<STR_LIT>':<EOL><INDENT>tokens.append(token)<EOL><DEDENT><DEDENT>Vars.start_pos = -<NUM_LIT:1> if t == '<STR_LIT:s>' else i<EOL><DEDENT>Vars.last_type = t<EOL><DEDENT>for i, char in enumerate(sentence):<EOL><INDENT>update(char, i)<EOL><DEDENT>update('<STR_LIT:U+0020>', len(sentence))<EOL>return tokens<EOL>

Converts a single sentence into a list of individual significant units
Args:
    sentence (str): Input string ie. 'This is a sentence.'
Returns:
    list<str>: List of tokens ie. ['this', 'is', 'a', 'sentence']


return SentenceTreeParser(sent).expand_parentheses()<EOL>

['1', '(', '2', '|', '3, ')'] -> [['1', '2'], ['1', '3']]
For example:

Will it (rain|pour) (today|tomorrow|)?

---->

Will it rain today?
Will it rain tomorrow?
Will it rain?
Will it pour today?
Will it pour tomorrow?
Will it pour?

Args:
    sent (list<str>): List of tokens in sentence
Returns:
    list<list<str>>: Multiple possible sentences from original


data = {}<EOL>for inp, out in zip(inputs, outputs):<EOL><INDENT>tup = tuple(inp)<EOL>if tup in data:<EOL><INDENT>data[tup].append(out)<EOL><DEDENT>else:<EOL><INDENT>data[tup] = [out]<EOL><DEDENT><DEDENT>inputs, outputs = [], []<EOL>for inp, outs in data.items():<EOL><INDENT>inputs.append(list(inp))<EOL>combined = [<NUM_LIT:0>] * len(outs[<NUM_LIT:0>])<EOL>for i in range(len(combined)):<EOL><INDENT>combined[i] = max(j[i] for j in outs)<EOL><DEDENT>outputs.append(combined)<EOL><DEDENT>return inputs, outputs<EOL>

Checks for duplicate inputs and if there are any,
remove one and set the output to the max of the two outputs
Args:
    inputs (list<list<float>>): Array of input vectors
    outputs (list<list<float>>): Array of output vectors
Returns:
    tuple<inputs, outputs>: The modified inputs and outputs


if '<STR_LIT::>' in name:<EOL><INDENT>parts = name.split('<STR_LIT::>')<EOL>intent_name, ent_name = parts[<NUM_LIT:0>], parts[<NUM_LIT:1>:]<EOL>return intent_name + '<STR_LIT>' + '<STR_LIT::>'.join(ent_name) + '<STR_LIT:}>'<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:{>' + name + '<STR_LIT:}>'<EOL><DEDENT>

Wraps SkillName:entity into SkillName:{entity}


self._tree = tree<EOL>

Construct a sentence tree fragment which is merely a wrapper for
a list of Strings

Args:
    tree (?): Base tree for the sentence fragment, type depends on
                subclass, refer to those subclasses


return self._tree<EOL>

Return the represented sentence tree as raw data.


return [[]]<EOL>

Expanded version of the fragment. In this case an empty sentence.

Returns:
    List<List<str>>: A list with an empty sentence (= token/string list)


return [[self._tree]]<EOL>

Creates one sentence that contains exactly that word.

Returns:
    List<List<str>>: A list with the given string as sentence
                        (= token/string list)


old_expanded = [[]]<EOL>for sub in self._tree:<EOL><INDENT>sub_expanded = sub.expand()<EOL>new_expanded = []<EOL>while len(old_expanded) > <NUM_LIT:0>:<EOL><INDENT>sentence = old_expanded.pop()<EOL>for new in sub_expanded:<EOL><INDENT>new_expanded.append(sentence + new)<EOL><DEDENT><DEDENT>old_expanded = new_expanded<EOL><DEDENT>return old_expanded<EOL>

Creates a combination of all sub-sentences.

Returns:
    List<List<str>>: A list with all subsentence expansions combined in
                        every possible way


options = []<EOL>for option in self._tree:<EOL><INDENT>options.extend(option.expand())<EOL><DEDENT>return options<EOL>

Returns all of its options as seperated sub-sentences.

Returns:
    List<List<str>>: A list containing the sentences created by all
                        expansions of its sub-sentences


self._current_position = <NUM_LIT:0><EOL>return self._parse_expr()<EOL>


<EOL>sentence_list = []<EOL>cur_sentence = []<EOL>sentence_list.append(Sentence(cur_sentence))<EOL>while self._current_position < len(self.tokens):<EOL><INDENT>cur = self.tokens[self._current_position]<EOL>self._current_position += <NUM_LIT:1><EOL>if cur == '<STR_LIT:(>':<EOL><INDENT>subexpr = self._parse_expr()<EOL>normal_brackets = False<EOL>if len(subexpr.tree()) == <NUM_LIT:1>:<EOL><INDENT>normal_brackets = True<EOL>cur_sentence.append(Word('<STR_LIT:(>'))<EOL><DEDENT>cur_sentence.append(subexpr)<EOL>if normal_brackets:<EOL><INDENT>cur_sentence.append(Word('<STR_LIT:)>'))<EOL><DEDENT><DEDENT>elif cur == '<STR_LIT:|>':<EOL><INDENT>cur_sentence = []<EOL>sentence_list.append(Sentence(cur_sentence))<EOL><DEDENT>elif cur == '<STR_LIT:)>':<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>cur_sentence.append(Word(cur))<EOL><DEDENT><DEDENT>return Options(sentence_list)<EOL>

Generate sentence token trees from the current position to
the next closing parentheses / end of the list and return it
['1', '(', '2', '|', '3, ')'] -> ['1', [['2'], ['3']]]
['2', '|', '3'] -> [['2'], ['3']]


return tree.expand()<EOL>

Expand a list of sub sentences to all combinated sentences.
['1', ['2', '3']] -> [['1', '2'], ['1', '3']]


@_save_args<EOL><INDENT>def add_intent(self, name, lines, reload_cache=False):<DEDENT>

self.intents.add(name, lines, reload_cache)<EOL>self.padaos.add_intent(name, lines)<EOL>self.must_train = True<EOL>

Creates a new intent, optionally checking the cache first

Args:
    name (str): The associated name of the intent
    lines (list<str>): All the sentences that should activate the intent
    reload_cache: Whether to ignore cached intent if exists


@_save_args<EOL><INDENT>def add_entity(self, name, lines, reload_cache=False):<DEDENT>

Entity.verify_name(name)<EOL>self.entities.add(Entity.wrap_name(name), lines, reload_cache)<EOL>self.padaos.add_entity(name, lines)<EOL>self.must_train = True<EOL>

Adds an entity that matches the given lines.

Example:
    self.add_intent('weather', ['will it rain on {weekday}?'])
    self.add_entity('{weekday}', ['monday', 'tuesday', 'wednesday'])  # ...

Args:
    name (str): The name of the entity
    lines (list<str>): Lines of example extracted entities
    reload_cache (bool): Whether to refresh all of cache


@_save_args<EOL><INDENT>def load_entity(self, name, file_name, reload_cache=False):<DEDENT>

Entity.verify_name(name)<EOL>self.entities.load(Entity.wrap_name(name), file_name, reload_cache)<EOL>with open(file_name) as f:<EOL><INDENT>self.padaos.add_entity(name, f.read().split('<STR_LIT:\n>'))<EOL><DEDENT>self.must_train = True<EOL>

Loads an entity, optionally checking the cache first

Args:
    name (str): The associated name of the entity
    file_name (str): The location of the entity file
    reload_cache (bool): Whether to refresh all of cache


@_save_args<EOL><INDENT>def load_file(self, *args, **kwargs):<DEDENT>

self.load_intent(*args, **kwargs)<EOL>


@_save_args<EOL><INDENT>def load_intent(self, name, file_name, reload_cache=False):<DEDENT>

self.intents.load(name, file_name, reload_cache)<EOL>with open(file_name) as f:<EOL><INDENT>self.padaos.add_intent(name, f.read().split('<STR_LIT:\n>'))<EOL><DEDENT>self.must_train = True<EOL>

Loads an intent, optionally checking the cache first

Args:
    name (str): The associated name of the intent
    file_name (str): The location of the intent file
    reload_cache (bool): Whether to refresh all of cache


self.intents.remove(name)<EOL>self.padaos.remove_intent(name)<EOL>self.must_train = True<EOL>


self.entities.remove(name)<EOL>self.padaos.remove_entity(name)<EOL>


def train(self, debug=True, force=False, single_thread=False, timeout=<NUM_LIT:20>):

if not self.must_train and not force:<EOL><INDENT>return<EOL><DEDENT>self.padaos.compile()<EOL>self.train_thread = Thread(target=self._train, kwargs=dict(<EOL>debug=debug,<EOL>single_thread=single_thread,<EOL>timeout=timeout<EOL>), daemon=True)<EOL>self.train_thread.start()<EOL>self.train_thread.join(timeout)<EOL>self.must_train = False<EOL>return not self.train_thread.is_alive()<EOL>

Trains all the loaded intents that need to be updated
If a cache file exists with the same hash as the intent file,
the intent will not be trained and just loaded from file

Args:
    debug (bool): Whether to print a message to stdout each time a new intent is trained
    force (bool): Whether to force training if already finished
    single_thread (bool): Whether to force running in a single thread
    timeout (float): Seconds before cancelling training
Returns:
    bool: True if training succeeded without timeout


def train_subprocess(self, *args, **kwargs):

ret = call([<EOL>sys.executable, '<STR_LIT>', '<STR_LIT>', '<STR_LIT:train>', self.cache_dir,<EOL>'<STR_LIT>', json.dumps(self.serialized_args),<EOL>'<STR_LIT>', json.dumps(args),<EOL>'<STR_LIT>', json.dumps(kwargs),<EOL>])<EOL>if ret == <NUM_LIT:2>:<EOL><INDENT>raise TypeError('<STR_LIT>'.format(args, kwargs))<EOL><DEDENT>data = self.serialized_args<EOL>self.clear()<EOL>self.apply_training_args(data)<EOL>self.padaos.compile()<EOL>if ret == <NUM_LIT:0>:<EOL><INDENT>self.must_train = False<EOL>return True<EOL><DEDENT>elif ret == <NUM_LIT:10>:  <EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(ret))<EOL><DEDENT>

Trains in a subprocess which provides a timeout guarantees everything shuts down properly

Args:
    See <train>
Returns:
    bool: True for success, False if timed out


if self.must_train:<EOL><INDENT>self.train()<EOL><DEDENT>intents = {} if self.train_thread and self.train_thread.is_alive() else {<EOL>i.name: i for i in self.intents.calc_intents(query, self.entities)<EOL>}<EOL>sent = tokenize(query)<EOL>for perfect_match in self.padaos.calc_intents(query):<EOL><INDENT>name = perfect_match['<STR_LIT:name>']<EOL>intents[name] = MatchData(name, sent, matches=perfect_match['<STR_LIT>'], conf=<NUM_LIT:1.0>)<EOL><DEDENT>return list(intents.values())<EOL>

Tests all the intents against the query and returns
data on how well each one matched against the query

Args:
    query (str): Input sentence to test against intents
Returns:
    list<MatchData>: List of intent matches
See calc_intent() for a description of the returned MatchData


matches = self.calc_intents(query)<EOL>if len(matches) == <NUM_LIT:0>:<EOL><INDENT>return MatchData('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>best_match = max(matches, key=lambda x: x.conf)<EOL>best_matches = (match for match in matches if match.conf == best_match.conf)<EOL>return min(best_matches, key=lambda x: sum(map(len, x.matches.values())))<EOL>

Tests all the intents against the query and returns
match data of the best intent

Args:
    query (str): Input sentence to test against intents
Returns:
    MatchData: Best intent match


def _train_and_save(obj, cache, data, print_updates):

obj.train(data)<EOL>if print_updates:<EOL><INDENT>print('<STR_LIT>' + obj.name + '<STR_LIT:.>')<EOL><DEDENT>obj.save(cache)<EOL>

Internal pickleable function used to train objects in another process


def wrapper(*args, **kwargs):<EOL><INDENT>np.random.seed(<NUM_LIT>)<EOL>return func(*args, **kwargs)<EOL><DEDENT>return wrapper<EOL>

Sets the seed of numpy's random number generator


def generate_sparse_system(n=<NUM_LIT:100>, m=<NUM_LIT:50>, p=<NUM_LIT:0.1>, eta=<NUM_LIT>, seed=<NUM_LIT>):

global x_true, x_obs, A<EOL>np.random.seed(seed)<EOL>x_true = <NUM_LIT:10> * np.random.randn(n) * (np.random.rand(n) < p)<EOL>A = np.random.randn(m, n)<EOL>y = A.dot(x_true)<EOL>xls = np.linalg.lstsq(A, y)[<NUM_LIT:0>]<EOL>ls_error = np.linalg.norm(xls - x_true, <NUM_LIT:2>)<EOL>return A, y, x_true, xls, ls_error<EOL>

Generate a sparse, noisy system (Ax = y)

    Parameters
    ----------
    n : int
        Number of variables

    m : int
        Number of observations

    p : float
        Probability of non-zero variable

    eta : float
        Noise level (standard deviation of additive Gaussian noise)

    seed : int
        Random seed

    Returns
    -------
    A : array_like
        Sensing matrix (m x n)

    y : array_like
        Observations (m,)

    x_true : array_like
        True sparse signal (n,)

    xls : array_like
        Least squares solution (n,)

    ls_error : float
        Error (2-norm) of the least squares solution


def generate_lowrank_matrix(n=<NUM_LIT:10>, m=<NUM_LIT:20>, k=<NUM_LIT:3>, eta=<NUM_LIT>, seed=<NUM_LIT>):

print("<STR_LIT>")<EOL>global Xtrue, Xobs<EOL>np.random.seed(seed)<EOL>Xtrue = np.sin(np.linspace(<NUM_LIT:0>, <NUM_LIT:2> * np.pi, n)).reshape(-<NUM_LIT:1>, <NUM_LIT:1>).dot(<EOL>np.cos(np.linspace(<NUM_LIT:0>, <NUM_LIT:2> * np.pi, m)).reshape(<NUM_LIT:1>, -<NUM_LIT:1>))<EOL>Xobs = Xtrue + eta * np.random.randn(n, m)<EOL>return Xobs, Xtrue<EOL>

Generate an n-by-m noisy low-rank matrix


class GradientOptimizer(Optimizer):<EOL><INDENT>@wraps(coro)<EOL>def __init__(self, *args, **kwargs):<EOL><INDENT>self.algorithm = coro(*args, **kwargs)<EOL>self.algorithm.send(None)<EOL>self.operators = []<EOL><DEDENT>def set_transform(self, func):<EOL><INDENT>self.transform = compose(destruct, func, self.restruct)<EOL><DEDENT>def minimize(self, f_df, x0, display=sys.stdout, maxiter=<NUM_LIT>):<EOL><INDENT>self.display = display<EOL>self.theta = x0<EOL>xk = self.algorithm.send(destruct(x0).copy())<EOL>store = defaultdict(list)<EOL>runtimes = []<EOL>if len(self.operators) == <NUM_LIT:0>:<EOL><INDENT>self.operators = [proxops.identity()]<EOL><DEDENT>obj, grad = wrap(f_df, x0)<EOL>transform = compose(destruct, *reversed(self.operators), self.restruct)<EOL>self.optional_print(tp.header(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']))<EOL>try:<EOL><INDENT>for k in count():<EOL><INDENT>tstart = perf_counter()<EOL>f = obj(xk)<EOL>df = grad(xk)<EOL>xk = transform(self.algorithm.send(df))<EOL>runtimes.append(perf_counter() - tstart)<EOL>store['<STR_LIT:f>'].append(f)<EOL>self.optional_print(tp.row([k,<EOL>f,<EOL>np.linalg.norm(destruct(df)),<EOL>tp.humantime(runtimes[-<NUM_LIT:1>])]))<EOL>if k >= maxiter:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT>self.optional_print(tp.bottom(<NUM_LIT:4>))<EOL>self.optional_print(u'<STR_LIT>'.format(store['<STR_LIT:f>'][-<NUM_LIT:1>]))<EOL>self.optional_print(u'<STR_LIT>'.format(tp.humantime(sum(runtimes))))<EOL>self.optional_print(u'<STR_LIT>'.format(<EOL>tp.humantime(np.mean(runtimes)),<EOL>tp.humantime(np.std(runtimes)),<EOL>))<EOL>return OptimizeResult({<EOL>'<STR_LIT:x>': self.restruct(xk),<EOL>'<STR_LIT:f>': f,<EOL>'<STR_LIT>': self.restruct(df),<EOL>'<STR_LIT:k>': k,<EOL>'<STR_LIT>': np.array(store['<STR_LIT:f>']),<EOL>})<EOL><DEDENT><DEDENT>return GradientOptimizer<EOL>

Turns a coroutine into a gradient based optimizer.


if isinstance(operator, str):<EOL><INDENT>op = getattr(proxops, operator)(*args)<EOL><DEDENT>elif isinstance(operator, proxops.ProximalOperatorBaseClass):<EOL><INDENT>op = operator<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.operators.append(op)<EOL>return self<EOL>

Adds a proximal operator to the list of operators


def __init__(self, tau=(<NUM_LIT>, <NUM_LIT>, <NUM_LIT>), tol=(<NUM_LIT>, <NUM_LIT>)):

self.operators = []<EOL>self.tau = namedtuple('<STR_LIT>', ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))(*tau)<EOL>self.tol = namedtuple('<STR_LIT>', ('<STR_LIT>', '<STR_LIT>'))(*tol)<EOL>

Proximal Consensus (ADMM)

Parameters
----------
tau : (float, float, float)
    ADMM scheduling. The augmented Lagrangian quadratic penalty parameter,
    rho, is initialized to tau[0]. Depending on the primal and dual residuals,
    the parameter is increased by a factor of tau[1] or decreased by a factor
    of tau[2] at every iteration. (See Boyd et. al. 2011 for details)

tol : (float, float)
    Primal and Dual residual tolerances


<EOL>xk = yield<EOL>vk = np.zeros_like(xk)<EOL>for k in count():<EOL><INDENT>grad = yield xk<EOL>vk = mom * vk - lr * grad<EOL>xk += vk<EOL><DEDENT>

Stochastic gradient descent (SGD)

Parameters
----------
lr : float, optional
    Learning rate (Default: 1e-3)

mom : float, optional
    Momentum (Default: 0.0)


xk = yield<EOL>yk = xk.copy()<EOL>for k in count():<EOL><INDENT>grad = yield yk<EOL>xprev = xk.copy()<EOL>xk = yk - lr * grad<EOL>yk = xk + (k / (k + <NUM_LIT>)) * (xk - xprev)<EOL><DEDENT>

Nesterov's accelerated gradient (NAG)

Parameters
----------
lr : float, optional
    Learning rate (Default: 1e-3)


@gradient_optimizer<EOL>def rmsprop(lr=<NUM_LIT>, damping=<NUM_LIT>, decay=<NUM_LIT>):

xk = yield<EOL>rms = np.zeros_like(xk)<EOL>for k in count():<EOL><INDENT>grad = yield xk<EOL>rms *= decay<EOL>rms += (<NUM_LIT:1> - decay) * grad**<NUM_LIT:2><EOL>xk -= lr * grad / (damping + np.sqrt(rms))<EOL><DEDENT>

RMSProp

Parameters
----------
lr : float, optional
    Learning rate (Default: 1e-3)

damping : float, optional
    Damping term (Default: 1e-12)

decay : float, optional
    Decay of the learning rate (Default: 0)


xk = yield<EOL>gradients = deque([], nterms)<EOL>for k in count():<EOL><INDENT>grad = yield xk<EOL>gradients.append(grad)<EOL>xk -= lr * np.mean(gradients, axis=<NUM_LIT:0>)<EOL><DEDENT>

Stochastic Average Gradient (SAG)

Parameters
----------
nterms : int, optional
    Number of gradient evaluations to use in the average (Default: 10)

lr : float, optional
    (Default: 1e-3)


xk = yield<EOL>mem = np.ones_like(xk)<EOL>g = np.zeros_like(xk)<EOL>g2 = np.zeros_like(xk)<EOL>for k in count():<EOL><INDENT>grad = yield xk<EOL>r = <NUM_LIT:1> / (mem + <NUM_LIT:1>)<EOL>r_1 = <NUM_LIT:1.> - r<EOL>g *= r_1<EOL>g += r * grad<EOL>g2 *= r_1<EOL>g2 += r * grad ** <NUM_LIT:2><EOL>glr = g ** <NUM_LIT:2> / (g2 + epsilon)<EOL>mem = <NUM_LIT:1> + mem * (<NUM_LIT:1> - glr)<EOL>xk -= grad * np.minimum(lr, glr) / (np.sqrt(g2) + epsilon)<EOL><DEDENT>

Squared mean over root mean squared cubed (SMORMS3)

Notes
-----
by Simon Funk
http://sifter.org/~simon/journal/20150420.html

Parameters
----------
lr : float, optional
    (Default: 1e-3)

epsilon : float, optional
    (Default: 1e-8)


@gradient_optimizer<EOL>def adam(lr=<NUM_LIT>, beta=(<NUM_LIT>, <NUM_LIT>), epsilon=<NUM_LIT>):

xk = yield<EOL>mk = np.zeros_like(xk)<EOL>vk = np.zeros_like(xk)<EOL>b1, b2 = beta<EOL>for k in count(start=<NUM_LIT:1>):<EOL><INDENT>grad = yield xk<EOL>mk *= b1<EOL>mk += (<NUM_LIT:1.> - b1) * grad<EOL>vk *= b2<EOL>vk += (<NUM_LIT:1.> - b2) * (grad ** <NUM_LIT:2>)<EOL>momentum_norm = mk / (<NUM_LIT:1> - b1 ** k)<EOL>velocity_norm = np.sqrt(vk / (<NUM_LIT:1> - b2 ** k))<EOL>xk -= lr * momentum_norm / (epsilon + velocity_norm)<EOL><DEDENT>

ADAM

Parameters
----------
lr : float, optional
    Learnin rate (Default: 1e-3)

beta : (float, float)
    (Default: (0.9, 0.999))

epsilon : float
    (Default: 1e-8)


@proxify<EOL>def nucnorm(x, rho, penalty, newshape=None):

orig_shape = x.shape<EOL>if newshape is not None:<EOL><INDENT>x = x.reshape(newshape)<EOL><DEDENT>u, s, v = np.linalg.svd(x, full_matrices=False)<EOL>sthr = np.maximum(s - (penalty / rho), <NUM_LIT:0>)<EOL>return np.linalg.multi_dot((u, np.diag(sthr), v)).reshape(orig_shape)<EOL>

Nuclear norm

Parameters
----------
penalty : float
    nuclear norm penalty hyperparameter

newshape : tuple, optional
    Desired shape of the parameters to apply the nuclear norm to. The given
    parameters are reshaped to an array with this shape, or not reshaped if
    the value of newshape is None. (Default: None)


@proxify<EOL>def sparse(x, rho, penalty):

lmbda = penalty / rho<EOL>return (x - lmbda) * (x >= lmbda) + (x + lmbda) * (x <= -lmbda)<EOL>

Proximal operator for the l1-norm: soft thresholding

Parameters
----------
penalty : float
    Strength or weight on the l1-norm


@proxify<EOL>def squared_error(x, rho, x_obs):

return (x + x_obs / rho) / (<NUM_LIT:1.> + <NUM_LIT:1.> / rho)<EOL>

Proximal operator for squared error (l2 or Fro. norm)

squared_error(x_obs)

Parameters
----------
x_obs : array_like
    Observed array or matrix that you want to stay close to


@proxify<EOL>def lbfgs(x, rho, f_df, maxiter=<NUM_LIT:20>):

def f_df_augmented(theta):<EOL><INDENT>f, df = f_df(theta)<EOL>obj = f + (rho / <NUM_LIT>) * np.linalg.norm(theta - x) ** <NUM_LIT:2><EOL>grad = df + rho * (theta - x)<EOL>return obj, grad<EOL><DEDENT>res = scipy_minimize(f_df_augmented, x, jac=True, method='<STR_LIT>',<EOL>options={'<STR_LIT>': maxiter, '<STR_LIT>': False})<EOL>return res.x<EOL>

Minimize the proximal operator of a given objective using L-BFGS

Parameters
----------
f_df : function
    Returns the objective and gradient of the function to minimize

maxiter : int
    Maximum number of L-BFGS iterations


return denoise_tv_bregman(x, rho / penalty)<EOL>

Total variation denoising proximal operator

Parameters
----------
penalty : float


return np.maximum(x, <NUM_LIT:0>)<EOL>

Projection onto the non-negative orthant


@proxify<EOL>def smooth(x, rho, penalty, axis=<NUM_LIT:0>, newshape=None):

orig_shape = x.shape<EOL>if newshape is not None:<EOL><INDENT>x = x.reshape(newshape)<EOL><DEDENT>n = x.shape[axis]<EOL>lap_op = spdiags([(<NUM_LIT:2> + rho / penalty) * np.ones(n),<EOL>-<NUM_LIT:1> * np.ones(n), -<NUM_LIT:1> * np.ones(n)],<EOL>[<NUM_LIT:0>, -<NUM_LIT:1>, <NUM_LIT:1>], n, n, format='<STR_LIT>')<EOL>A = penalty * lap_op<EOL>b = rho * np.rollaxis(x, axis, <NUM_LIT:0>)<EOL>return np.rollaxis(spsolve(A, b), axis, <NUM_LIT:0>).reshape(orig_shape)<EOL>

Applies a smoothing operator along one dimension

currently only accepts a matrix as input

Parameters
----------
penalty : float

axis : int, optional
    Axis along which to apply the smoothing (Default: 0)

newshape : tuple, optional
    Desired shape of the parameters to apply the nuclear norm to. The given
    parameters are reshaped to an array with this shape, or not reshaped if
    the value of newshape is None. (Default: None)


U, V = np.linalg.eigh(x)<EOL>return V.dot(np.diag(np.maximum(U, <NUM_LIT:0>)).dot(V.T))<EOL>


@proxify<EOL>def linear(x, rho, weights):

return x - weights / rho<EOL>

Proximal operator for a linear function w^T x


<EOL>u = np.flipud(np.sort(x.ravel()))<EOL>lambdas = (<NUM_LIT:1> - np.cumsum(u)) / (<NUM_LIT:1.> + np.arange(u.size))<EOL>ix = np.where(u + lambdas > <NUM_LIT:0>)[<NUM_LIT:0>].max()<EOL>return np.maximum(x + lambdas[ix], <NUM_LIT:0>)<EOL>

Projection onto the probability simplex

http://arxiv.org/pdf/1309.1541v1.pdf


@proxify<EOL>def columns(x, rho, proxop):

xnext = np.zeros_like(x)<EOL>for ix in range(x.shape[<NUM_LIT:1>]):<EOL><INDENT>xnext[:, ix] = proxop(x[:, ix], rho)<EOL><DEDENT>return xnext<EOL>

Applies a proximal operator to the columns of a matrix


return x<EOL>


@proxify<EOL>def fantope(x, rho, dim, tol=<NUM_LIT>):

U, V = np.linalg.eigh(x)<EOL>minval, maxval = np.maximum(U.min(), <NUM_LIT:0>), np.maximum(U.max(), <NUM_LIT:20> * dim)<EOL>while True:<EOL><INDENT>theta = <NUM_LIT:0.5> * (maxval + minval)<EOL>thr_eigvals = np.minimum(np.maximum((U - theta), <NUM_LIT:0>), <NUM_LIT:1>)<EOL>constraint = np.sum(thr_eigvals)<EOL>if np.abs(constraint - dim) <= tol:<EOL><INDENT>break<EOL><DEDENT>elif constraint < dim:<EOL><INDENT>maxval = theta<EOL><DEDENT>elif constraint > dim:<EOL><INDENT>minval = theta<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return np.linalg.multi_dot((V, np.diag(thr_eigvals), V.T))<EOL>

Projection onto the fantope [1]_

.. [1] Vu, Vincent Q., et al. "Fantope projection and selection: A
       near-optimal convex relaxation of sparse PCA." Advances in
       neural information processing systems. 2013.


self.P = A.T.dot(A)<EOL>self.q = A.T.dot(b)<EOL>self.n = self.q.size<EOL>

Proximal operator for solving a linear least squares system, Ax = b

Parameters
----------
A : array_like
    Sensing matrix (Ax = b)

b : array_like
    Responses (Ax = b)


def objective(param_scales=(<NUM_LIT:1>, <NUM_LIT:1>), xstar=None, seed=None):

ndim = len(param_scales)<EOL>def decorator(func):<EOL><INDENT>@wraps(func)<EOL>def wrapper(theta):<EOL><INDENT>return func(theta)<EOL><DEDENT>def param_init():<EOL><INDENT>np.random.seed(seed)<EOL>return np.random.randn(ndim,) * np.array(param_scales)<EOL><DEDENT>wrapper.ndim = ndim<EOL>wrapper.param_init = param_init<EOL>wrapper.xstar = xstar<EOL>return wrapper<EOL><DEDENT>return decorator<EOL>

Gives objective functions a number of dimensions and parameter range

    Parameters
    ----------
    param_scales : (int, int)
        Scale (std. dev.) for choosing each parameter

    xstar : array_like
        Optimal parameters


k0, k1, depth = <NUM_LIT>, <NUM_LIT:100>, <NUM_LIT:0.5><EOL>shallow = <NUM_LIT:0.5> * k0 * theta ** <NUM_LIT:2> + depth<EOL>deep = <NUM_LIT:0.5> * k1 * theta ** <NUM_LIT:2><EOL>obj = float(np.minimum(shallow, deep))<EOL>grad = np.where(deep < shallow, k1 * theta, k0 * theta)<EOL>return obj, grad<EOL>

Pointwise minimum of two quadratic bowls


x, y = theta<EOL>obj = (<NUM_LIT:1> - x)**<NUM_LIT:2> + <NUM_LIT:100> * (y - x**<NUM_LIT:2>)**<NUM_LIT:2><EOL>grad = np.zeros(<NUM_LIT:2>)<EOL>grad[<NUM_LIT:0>] = <NUM_LIT:2> * x - <NUM_LIT> * (x * y - x**<NUM_LIT:3>) - <NUM_LIT:2><EOL>grad[<NUM_LIT:1>] = <NUM_LIT:200> * (y - x**<NUM_LIT:2>)<EOL>return obj, grad<EOL>

Objective and gradient for the rosenbrock function


return <NUM_LIT:0.5> * np.linalg.norm(theta)**<NUM_LIT:2>, theta<EOL>


x, y = theta<EOL>obj = <NUM_LIT> * (x ** <NUM_LIT:2> + y ** <NUM_LIT:2>) - <NUM_LIT> * x * y<EOL>grad = np.array([<NUM_LIT> * x - <NUM_LIT> * y, <NUM_LIT> * y - <NUM_LIT> * x])<EOL>return obj, grad<EOL>


x, y = theta<EOL>A = <NUM_LIT> - x + x * y<EOL>B = <NUM_LIT> - x + x * y**<NUM_LIT:2><EOL>C = <NUM_LIT> - x + x * y**<NUM_LIT:3><EOL>obj = A ** <NUM_LIT:2> + B ** <NUM_LIT:2> + C ** <NUM_LIT:2><EOL>grad = np.array([<EOL><NUM_LIT:2> * A * (y - <NUM_LIT:1>) + <NUM_LIT:2> * B * (y ** <NUM_LIT:2> - <NUM_LIT:1>) + <NUM_LIT:2> * C * (y ** <NUM_LIT:3> - <NUM_LIT:1>),<EOL><NUM_LIT:2> * A * x + <NUM_LIT:4> * B * x * y + <NUM_LIT:6> * C * x * y ** <NUM_LIT:2><EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>A = x + <NUM_LIT:2> * y - <NUM_LIT:7><EOL>B = <NUM_LIT:2> * x + y - <NUM_LIT:5><EOL>obj = A**<NUM_LIT:2> + B**<NUM_LIT:2><EOL>grad = np.array([<NUM_LIT:2> * A + <NUM_LIT:4> * B, <NUM_LIT:4> * A + <NUM_LIT:2> * B])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = np.sin(x + y) + (x - y)**<NUM_LIT:2> - <NUM_LIT> * x + <NUM_LIT> * y + <NUM_LIT:1><EOL>grad = np.array([np.cos(x + y) + <NUM_LIT:2> * (x - y) - <NUM_LIT>,<EOL>np.cos(x + y) - <NUM_LIT:2> * (x - y) + <NUM_LIT>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = <NUM_LIT:2> * x ** <NUM_LIT:2> - <NUM_LIT> * x ** <NUM_LIT:4> + x ** <NUM_LIT:6> / <NUM_LIT:6> + x * y + y ** <NUM_LIT:2><EOL>grad = np.array([<EOL><NUM_LIT:4> * x - <NUM_LIT> * x ** <NUM_LIT:3> + x ** <NUM_LIT:5> + y,<EOL>x + <NUM_LIT:2> * y<EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = - np.sin(x) * np.sin(x ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20> -np.sin(y) * np.sin(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20><EOL>grad = np.array([<EOL>- np.cos(x) * np.sin(x ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20> - (<NUM_LIT> / np.pi) * x *<EOL>np.sin(x) * np.sin(x ** <NUM_LIT:2> / np.pi) ** <NUM_LIT> * np.cos(x ** <NUM_LIT:2> / np.pi),<EOL>- np.cos(y) * np.sin(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20> - (<NUM_LIT> / np.pi) * y * np.sin(y) *<EOL>np.sin(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi) ** <NUM_LIT> * np.cos(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi),<EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = x ** <NUM_LIT:2> + <NUM_LIT:2> * y ** <NUM_LIT:2> - <NUM_LIT> * np.cos(<NUM_LIT:3> * np.pi * x) - <NUM_LIT> * np.cos(<NUM_LIT:4> * np.pi * y) + <NUM_LIT><EOL>grad = np.array([<EOL><NUM_LIT:2> * x + <NUM_LIT> * np.sin(<NUM_LIT:3> * np.pi * x) * <NUM_LIT:3> * np.pi,<EOL><NUM_LIT:4> * y + <NUM_LIT> * np.sin(<NUM_LIT:4> * np.pi * y) * <NUM_LIT:4> * np.pi,<EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = x ** <NUM_LIT:2> + y ** <NUM_LIT:2> + (<NUM_LIT:0.5> * x + y) ** <NUM_LIT:2> + (<NUM_LIT:0.5> * x + y) ** <NUM_LIT:4><EOL>grad = np.array([<EOL><NUM_LIT> * x + y + <NUM_LIT:2> * (<NUM_LIT:0.5> * x + y) ** <NUM_LIT:3>,<EOL><NUM_LIT:4> * y + x + <NUM_LIT:4> * (<NUM_LIT:0.5> * x + y) ** <NUM_LIT:3>,<EOL>])<EOL>return obj, grad<EOL>


@objective(xstar=(<NUM_LIT:1>, <NUM_LIT:1> / np.sqrt(<NUM_LIT:2>)))<EOL>def dixon_price(theta):

x, y = theta<EOL>obj = (x - <NUM_LIT:1>) ** <NUM_LIT:2> + <NUM_LIT:2> * (<NUM_LIT:2> * y ** <NUM_LIT:2> - x) ** <NUM_LIT:2><EOL>grad = np.array([<EOL><NUM_LIT:2> * x - <NUM_LIT:2> - <NUM_LIT:4> * (<NUM_LIT:2> * y ** <NUM_LIT:2> - x),<EOL><NUM_LIT:16> * (<NUM_LIT:2> * y ** <NUM_LIT:2> - x) * y,<EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = (<NUM_LIT:1> + (x + y + <NUM_LIT:1>) ** <NUM_LIT:2> * (<NUM_LIT> - <NUM_LIT> * x + <NUM_LIT:3> * x ** <NUM_LIT:2> - <NUM_LIT> * y + <NUM_LIT:6> * x * y + <NUM_LIT:3> * y ** <NUM_LIT:2>)) *(<NUM_LIT:30> + (<NUM_LIT:2> * x - <NUM_LIT:3> * y) ** <NUM_LIT:2> *<EOL>(<NUM_LIT> - <NUM_LIT:32> * x + <NUM_LIT:12> * x ** <NUM_LIT:2> + <NUM_LIT> * y - <NUM_LIT> * x * y + <NUM_LIT> * x ** <NUM_LIT:2>))<EOL>grad = np.array([<EOL>((<NUM_LIT:2> * x - <NUM_LIT:3> * y)**<NUM_LIT:2> * (<NUM_LIT> * x - <NUM_LIT> * y - <NUM_LIT:32>) + (<NUM_LIT:8> * x - <NUM_LIT:12> * y) *<EOL>(<NUM_LIT> * x**<NUM_LIT:2> - <NUM_LIT> * x * y - <NUM_LIT:32> * x + <NUM_LIT> * y + <NUM_LIT>)) *<EOL>((x + y + <NUM_LIT:1>)**<NUM_LIT:2> * (<NUM_LIT:3> * x**<NUM_LIT:2> + <NUM_LIT:6> * x * y - <NUM_LIT> * x + <NUM_LIT:3> * y**<NUM_LIT:2> - <NUM_LIT> * y + <NUM_LIT>) + <NUM_LIT:1>) +<EOL>((<NUM_LIT:2> * x - <NUM_LIT:3> * y)**<NUM_LIT:2> * (<NUM_LIT> * x**<NUM_LIT:2> - <NUM_LIT> * x * y - <NUM_LIT:32> * x + <NUM_LIT> * y + <NUM_LIT>) + <NUM_LIT:30>) *<EOL>((x + y + <NUM_LIT:1>)**<NUM_LIT:2> *<EOL>(<NUM_LIT:6> * x + <NUM_LIT:6> * y - <NUM_LIT>) + (<NUM_LIT:2> * x + <NUM_LIT:2> * y + <NUM_LIT:2>) *<EOL>(<NUM_LIT:3> * x**<NUM_LIT:2> + <NUM_LIT:6> * x * y - <NUM_LIT> * x + <NUM_LIT:3> * y**<NUM_LIT:2> - <NUM_LIT> * y + <NUM_LIT>)),<EOL>((-<NUM_LIT> * x + <NUM_LIT>) * (<NUM_LIT:2> * x - <NUM_LIT:3> * y)**<NUM_LIT:2> + (-<NUM_LIT:12> * x + <NUM_LIT> * y) *<EOL>(<NUM_LIT> * x**<NUM_LIT:2> - <NUM_LIT> * x * y - <NUM_LIT:32> * x + <NUM_LIT> * y + <NUM_LIT>)) *<EOL>((x + y + <NUM_LIT:1>)**<NUM_LIT:2> * (<NUM_LIT:3> * x**<NUM_LIT:2> + <NUM_LIT:6> * x * y - <NUM_LIT> * x + <NUM_LIT:3> * y**<NUM_LIT:2> - <NUM_LIT> * y + <NUM_LIT>) + <NUM_LIT:1>) +<EOL>((<NUM_LIT:2> * x - <NUM_LIT:3> * y)**<NUM_LIT:2> * (<NUM_LIT> * x**<NUM_LIT:2> - <NUM_LIT> * x * y - <NUM_LIT:32> * x + <NUM_LIT> * y + <NUM_LIT>) + <NUM_LIT:30>) *<EOL>((x + y + <NUM_LIT:1>)**<NUM_LIT:2> * (<NUM_LIT:6> * x + <NUM_LIT:6> * y - <NUM_LIT>) + (<NUM_LIT:2> * x + <NUM_LIT:2> * y + <NUM_LIT:2>) *<EOL>(<NUM_LIT:3> * x**<NUM_LIT:2> + <NUM_LIT:6> * x * y - <NUM_LIT> * x + <NUM_LIT:3> * y**<NUM_LIT:2> - <NUM_LIT> * y + <NUM_LIT>)),<EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = <NUM_LIT:0.5> * (x ** <NUM_LIT:4> - <NUM_LIT:16> * x ** <NUM_LIT:2> + <NUM_LIT:5> * x + y ** <NUM_LIT:4> - <NUM_LIT:16> * y ** <NUM_LIT:2> + <NUM_LIT:5> * y)<EOL>grad = np.array([<EOL><NUM_LIT:2> * x ** <NUM_LIT:3> - <NUM_LIT:16> * x + <NUM_LIT>,<EOL><NUM_LIT:2> * y ** <NUM_LIT:3> - <NUM_LIT:16> * y + <NUM_LIT>,<EOL>])<EOL>return obj, grad<EOL>


memoized_f_df = lrucache(lambda x: f_df(restruct(x, xref)), size)<EOL>objective = compose(first, memoized_f_df)<EOL>gradient = compose(destruct, second, memoized_f_df)<EOL>return objective, gradient<EOL>

Memoizes an objective + gradient function, and splits it into
two functions that return just the objective and gradient, respectively.

Parameters
----------
f_df : function
    Must be unary (takes a single argument)

xref : list, dict, or array_like
    The form of the parameters

size : int, optional
    Size of the cache (Default=1)


def decorator(func):<EOL><INDENT>@wraps(func)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>wrapper.__doc__ = docstr<EOL>return wrapper<EOL><DEDENT>return decorator<EOL>

Decorates a function with the given docstring

Parameters
----------
docstr : string


if size == <NUM_LIT:0>:<EOL><INDENT>return func<EOL><DEDENT>elif size < <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not is_arity(<NUM_LIT:1>, func):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>cache = OrderedDict()<EOL>def wrapper(x):<EOL><INDENT>if not(type(x) is np.ndarray):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if x.size <= <NUM_LIT>:<EOL><INDENT>key = hash(x.tostring())<EOL><DEDENT>else:<EOL><INDENT>key = hash(repr(x))<EOL><DEDENT>if key not in cache:<EOL><INDENT>if len(cache) >= size:<EOL><INDENT>cache.popitem(last=False)<EOL><DEDENT>cache[key] = func(x)<EOL><DEDENT>return cache[key]<EOL><DEDENT>return wrapper<EOL>

A simple implementation of a least recently used (LRU) cache.
Memoizes the recent calls of a computationally intensive function.

Parameters
----------
func : function
    Must be unary (takes a single argument)

size : int
    The size of the cache (number of previous calls to store)


def check_grad(f_df, xref, stepsize=<NUM_LIT>, tol=<NUM_LIT>, width=<NUM_LIT:15>, style='<STR_LIT>', out=sys.stdout):

CORRECT = u'<STR_LIT>'<EOL>INCORRECT = u'<STR_LIT>'<EOL>obj, grad = wrap(f_df, xref, size=<NUM_LIT:0>)<EOL>x0 = destruct(xref)<EOL>df = grad(x0)<EOL>out.write(tp.header(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"], width=width, style=style) + "<STR_LIT:\n>")<EOL>out.flush()<EOL>def parse_error(number):<EOL><INDENT>failure = "<STR_LIT>"<EOL>passing = "<STR_LIT>"<EOL>warning = "<STR_LIT>"<EOL>end = "<STR_LIT>"<EOL>base = "<STR_LIT>"<EOL>if error < <NUM_LIT:0.1> * tol:<EOL><INDENT>return base.format(passing, error, end)<EOL><DEDENT>elif error < tol:<EOL><INDENT>return base.format(warning, error, end)<EOL><DEDENT>else:<EOL><INDENT>return base.format(failure, error, end)<EOL><DEDENT><DEDENT>num_errors = <NUM_LIT:0><EOL>for j in range(x0.size):<EOL><INDENT>dx = np.zeros(x0.size)<EOL>dx[j] = stepsize<EOL>df_approx = (obj(x0 + dx) - obj(x0 - dx)) / (<NUM_LIT:2> * stepsize)<EOL>df_analytic = df[j]<EOL>abs_error = np.linalg.norm(df_approx - df_analytic)<EOL>error = abs_error if np.allclose(abs_error, <NUM_LIT:0>) else abs_error /(np.linalg.norm(df_analytic) + np.linalg.norm(df_approx))<EOL>num_errors += error >= tol<EOL>errstr = CORRECT if error < tol else INCORRECT<EOL>out.write(tp.row([df_approx, df_analytic, parse_error(error) + '<STR_LIT:U+0020>' + errstr],<EOL>width=width, style=style) + "<STR_LIT:\n>")<EOL>out.flush()<EOL><DEDENT>out.write(tp.bottom(<NUM_LIT:3>, width=width, style=style) + "<STR_LIT:\n>")<EOL>return num_errors<EOL>

Compares the numerical gradient to the analytic gradient

Parameters
----------
f_df : function
    The analytic objective and gradient function to check

x0 : array_like
    Parameter values to check the gradient at

stepsize : float, optional
    Stepsize for the numerical gradient. Too big and this will poorly estimate the gradient.
    Too small and you will run into precision issues (default: 1e-6)

tol : float, optional
    Tolerance to use when coloring correct/incorrect gradients (default: 1e-5)

width : int, optional
    Width of the table columns (default: 15)

style : string, optional
    Style of the printed table, see tableprint for a list of styles (default: 'round')


try:<EOL><INDENT>return psycopg2.connect(**db_state['<STR_LIT>'])<EOL><DEDENT>except psycopg2.OperationalError as exc:<EOL><INDENT>nosuch_db = '<STR_LIT>' % db_state['<STR_LIT>']['<STR_LIT>']<EOL>if nosuch_db in str(exc):<EOL><INDENT>try:<EOL><INDENT>master = psycopg2.connect(database='<STR_LIT>')<EOL>master.rollback()<EOL>master.autocommit = True<EOL>cursor = master.cursor()<EOL>cursor.execute('<STR_LIT>' % db_state['<STR_LIT>']['<STR_LIT>'])<EOL>cursor.close()<EOL>master.close()<EOL><DEDENT>except psycopg2.Error as exc:<EOL><INDENT>message = ('<STR_LIT>'<EOL>+ db_state['<STR_LIT>']['<STR_LIT>']<EOL>+ '<STR_LIT>' % exc)<EOL>raise RuntimeError(message)<EOL><DEDENT>else:<EOL><INDENT>conn = psycopg2.connect(**db_state['<STR_LIT>'])<EOL>db_state['<STR_LIT>'] = True<EOL>return conn<EOL><DEDENT><DEDENT><DEDENT>


if not db_state['<STR_LIT>']:<EOL><INDENT>return<EOL><DEDENT>get_conn().close()<EOL>master = psycopg2.connect(database='<STR_LIT>')<EOL>master.rollback()<EOL>master.autocommit = True<EOL>cursor = master.cursor()<EOL>cursor.execute('<STR_LIT>' % db_state['<STR_LIT>']['<STR_LIT>'])<EOL>cursor.close()<EOL>master.close()<EOL>


def __init__(self, connection, table, xform):

super(RenameReplace, self).__init__(connection, table)<EOL>self.xform = xform<EOL>

xform must be a function which translates old
names to new ones, used on tables & pk constraints


<EOL>dt = util.to_utc(dt)<EOL>unix_timestamp = calendar.timegm(dt.timetuple())<EOL>val = ((unix_timestamp - psql_epoch) * <NUM_LIT>) + dt.microsecond<EOL>return ('<STR_LIT>', (<NUM_LIT:8>, val))<EOL>


try:<EOL><INDENT>nt = n.as_tuple()<EOL><DEDENT>except AttributeError:<EOL><INDENT>raise TypeError('<STR_LIT>' % n)<EOL><DEDENT>digits = []<EOL>if isinstance(nt.exponent, str):<EOL><INDENT>ndigits = <NUM_LIT:0><EOL>weight = <NUM_LIT:0><EOL>sign = <NUM_LIT><EOL>dscale = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>decdigits = list(reversed(nt.digits + (nt.exponent % <NUM_LIT:4>) * (<NUM_LIT:0>,)))<EOL>weight = <NUM_LIT:0><EOL>while decdigits:<EOL><INDENT>if any(decdigits[:<NUM_LIT:4>]):<EOL><INDENT>break<EOL><DEDENT>weight += <NUM_LIT:1><EOL>del decdigits[:<NUM_LIT:4>]<EOL><DEDENT>while decdigits:<EOL><INDENT>digits.insert(<NUM_LIT:0>, ndig(decdigits[:<NUM_LIT:4>]))<EOL>del decdigits[:<NUM_LIT:4>]<EOL><DEDENT>ndigits = len(digits)<EOL>weight += nt.exponent // <NUM_LIT:4> + ndigits - <NUM_LIT:1><EOL>sign = nt.sign * <NUM_LIT><EOL>dscale = -min(<NUM_LIT:0>, nt.exponent)<EOL><DEDENT>data = [ndigits, weight, sign, dscale] + digits<EOL>return ('<STR_LIT>' % ndigits, [<NUM_LIT:2> * len(data)] + data)<EOL>

NBASE = 1000
ndigits = total number of base-NBASE digits
weight = base-NBASE weight of first digit
sign = 0x0000 if positive, 0x4000 if negative, 0xC000 if nan
dscale = decimal digits after decimal place


return p.startswith('<STR_LIT:[>')<EOL>

Is an argument of the URL.

>>> _is_url_arg('[idAction]')
True
>>> _is_url_arg('actions')
False


return line.split('<STR_LIT:U+0020>', <NUM_LIT:1>)[<NUM_LIT:0>] in HTTP_METHODS<EOL>

Is a definition of a Trello endpoint.

>>> _is_api_definition('GET /1/actions/[idAction]')
True
>>> _is_api_definition('action')
False


def upper2underscore(text):<EOL><INDENT>for char in text:<EOL><INDENT>if char.islower():<EOL><INDENT>yield char<EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT:_>'<EOL>if char.isalpha():<EOL><INDENT>yield char.lower()<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return '<STR_LIT>'.join(upper2underscore(url))<EOL>

Translate camelCase into underscore format.

>>> _camelcase_to_underscore('minutesBetweenSummaries')
'minutes_between_summaries'


tree = {}<EOL>for method, url, doc in endpoints:<EOL><INDENT>path = [p for p in url.strip('<STR_LIT:/>').split('<STR_LIT:/>')]<EOL>here = tree<EOL>version = path[<NUM_LIT:0>]<EOL>here.setdefault(version, {})<EOL>here = here[version]<EOL>for p in path[<NUM_LIT:1>:]:<EOL><INDENT>part = _camelcase_to_underscore(p)<EOL>here.setdefault(part, {})<EOL>here = here[part]<EOL><DEDENT>if not '<STR_LIT>' in here:<EOL><INDENT>here['<STR_LIT>'] = [[method, doc]]<EOL><DEDENT>else:<EOL><INDENT>if not method in here['<STR_LIT>']:<EOL><INDENT>here['<STR_LIT>'].append([method, doc])<EOL><DEDENT><DEDENT><DEDENT>return tree<EOL>

Creates the Trello endpoint tree.

>>> r = {'1': { \
             'actions': {'METHODS': {'GET'}}, \
             'boards': { \
                 'members': {'METHODS': {'DELETE'}}}} \
        }
>>> r == create_tree([ \
             'GET /1/actions/[idAction]', \
             'DELETE /1/boards/[board_id]/members/[idMember]'])
True


ep = requests.get(TRELLO_API_DOC).content<EOL>root = html.fromstring(ep)<EOL>links = root.xpath('<STR_LIT>')<EOL>pages = [requests.get(TRELLO_API_DOC + u)<EOL>for u in links if u.endswith('<STR_LIT>')]<EOL>endpoints = []<EOL>for page in pages:<EOL><INDENT>root = html.fromstring(page.content)<EOL>sections = root.xpath('<STR_LIT>')<EOL>for sec in sections:<EOL><INDENT>ep_html = etree.tostring(sec).decode('<STR_LIT:utf-8>')<EOL>ep_text = html2text(ep_html).splitlines()<EOL>match = EP_DESC_REGEX.match(ep_text[<NUM_LIT:0>])<EOL>if not match:<EOL><INDENT>continue<EOL><DEDENT>ep_method, ep_url = match.groups()<EOL>ep_text[<NUM_LIT:0>] = '<STR_LIT:U+0020>'.join([ep_method, ep_url])<EOL>ep_doc = b64encode(gzip.compress('<STR_LIT:\n>'.join(ep_text).encode('<STR_LIT:utf-8>')))<EOL>endpoints.append((ep_method, ep_url, ep_doc))<EOL><DEDENT><DEDENT>print(yaml.dump(create_tree(endpoints)))<EOL>


def get_partial_api(key, token=None):<EOL><INDENT>return TrelloAPI(ENDPOINTS[version], version, key, token=token)<EOL><DEDENT>get_partial_api.__doc__ ="""<STR_LIT>""".format(version)<EOL>return get_partial_api<EOL>

Generates a factory function to instantiate the API with the given
version.


if self._api_arg:<EOL><INDENT>mypart = str(self._api_arg)<EOL><DEDENT>else:<EOL><INDENT>mypart = self._name<EOL><DEDENT>if self._parent:<EOL><INDENT>return '<STR_LIT:/>'.join(filter(None, [self._parent._url, mypart]))<EOL><DEDENT>else:<EOL><INDENT>return mypart<EOL><DEDENT>

Resolve the URL to this point.

>>> trello = TrelloAPIV1('APIKEY')
>>> trello.batch._url
'1/batch'
>>> trello.boards(board_id='BOARD_ID')._url
'1/boards/BOARD_ID'
>>> trello.boards(board_id='BOARD_ID')(field='FIELD')._url
'1/boards/BOARD_ID/FIELD'
>>> trello.boards(board_id='BOARD_ID').cards(filter='FILTER')._url
'1/boards/BOARD_ID/cards/FILTER'


def _api_call(self, method_name, *args, **kwargs):

params = kwargs.setdefault('<STR_LIT>', {})<EOL>params.update({'<STR_LIT:key>': self._apikey})<EOL>if self._token is not None:<EOL><INDENT>params.update({'<STR_LIT>': self._token})<EOL><DEDENT>http_method = getattr(requests, method_name)<EOL>return http_method(TRELLO_URL + self._url, *args, **kwargs)<EOL>