params = {}<EOL>params_general = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": <NUM_LIT>,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": "<STR_LIT:file>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": <NUM_LIT>,  <EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": <NUM_LIT:5>,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": <NUM_LIT:1000>,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": <NUM_LIT:100>,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": ALIASES_JSON,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": MENU_YAML,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_projects = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": PROJECTS_JSON,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_phases = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>general_config_params = [params_general, params_projects, params_phases]<EOL>for section_params in general_config_params:<EOL><INDENT>params.update(section_params)<EOL><DEDENT>params_collection = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT:password>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:user>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:url>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_enrichment = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT:url>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": <NUM_LIT:2>,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:user>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:password>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_panels = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT:strict>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_report = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": [],<EOL>"<STR_LIT:type>": list,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_sortinghat = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT:True>",<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": ["<STR_LIT:email>"],<EOL>"<STR_LIT:type>": list,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": <NUM_LIT>,<EOL>"<STR_LIT:type>": int,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:host>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:user>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT:root>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT:password>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT:type>": list,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": True,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>"<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": [],<EOL>"<STR_LIT:type>": list,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": None,<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": [],<EOL>"<STR_LIT:type>": list,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": [],<EOL>"<STR_LIT:type>": list,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT:default>": False,<EOL>"<STR_LIT:type>": bool,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>params_track_items = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>},<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": False,<EOL>"<STR_LIT:default>": "<STR_LIT>",<EOL>"<STR_LIT:type>": str,<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>tasks_config_params = [params_collection, params_enrichment, params_panels,<EOL>params_report, params_sortinghat, params_track_items]<EOL>for section_params in tasks_config_params:<EOL><INDENT>params.update(section_params)<EOL><DEDENT>return params<EOL>


def __create_arthur_json(self, repo, backend_args):

backend_args = self._compose_arthur_params(self.backend_section, repo)<EOL>if self.backend_section == '<STR_LIT>':<EOL><INDENT>backend_args['<STR_LIT>'] = os.path.join(self.REPOSITORY_DIR, repo)<EOL><DEDENT>backend_args['<STR_LIT>'] = self.backend_tag(repo)<EOL>ajson = {"<STR_LIT>": [{}]}<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = self.backend_tag(repo)<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = self.backend_section.split("<STR_LIT::>")[<NUM_LIT:0>]<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = backend_args<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = backend_args['<STR_LIT>']<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = {}<EOL>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>'] = {"<STR_LIT>": self.ARTHUR_TASK_DELAY}<EOL>es_col_url = self._get_collection_url()<EOL>es_index = self.conf[self.backend_section]['<STR_LIT>']<EOL>es = ElasticSearch(es_col_url, es_index)<EOL>connector = get_connector_from_name(self.backend_section)<EOL>klass = connector[<NUM_LIT:0>]  <EOL>signature = inspect.signature(klass.fetch)<EOL>last_activity = None<EOL>filter_ = {"<STR_LIT:name>": "<STR_LIT>", "<STR_LIT:value>": backend_args['<STR_LIT>']}<EOL>if '<STR_LIT>' in signature.parameters:<EOL><INDENT>last_activity = es.get_last_item_field('<STR_LIT>', [filter_])<EOL>if last_activity:<EOL><INDENT>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>']['<STR_LIT>'] = last_activity.isoformat()<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in signature.parameters:<EOL><INDENT>last_activity = es.get_last_item_field('<STR_LIT>', [filter_])<EOL>if last_activity:<EOL><INDENT>ajson["<STR_LIT>"][<NUM_LIT:0>]['<STR_LIT>']['<STR_LIT>'] = last_activity<EOL><DEDENT><DEDENT>if last_activity:<EOL><INDENT>logging.info("<STR_LIT>", last_activity)<EOL><DEDENT>return(ajson)<EOL>

Create the JSON for configuring arthur to collect data

        https://github.com/grimoirelab/arthur#adding-tasks
        Sample for git:

        {
        "tasks": [
            {
                "task_id": "arthur.git",
                "backend": "git",
                "backend_args": {
                    "gitpath": "/tmp/arthur_git/",
                    "uri": "https://github.com/grimoirelab/arthur.git"
                },
                "category": "commit",
                "archive_args": {
                    "archive_path": '/tmp/test_archives',
                    "fetch_from_archive": false,
                    "archive_after": None
                },
                "scheduler_args": {
                    "delay": 10
                }
            }
        ]
        }


with self.ARTHUR_FEED_LOCK:<EOL><INDENT>if (time.time() - self.ARTHUR_LAST_MEMORY_CHECK) > <NUM_LIT:5> * self.ARTHUR_LAST_MEMORY_CHECK_TIME:<EOL><INDENT>self.ARTHUR_LAST_MEMORY_CHECK = time.time()<EOL>logger.debug("<STR_LIT>")<EOL>try:<EOL><INDENT>memory_size = self.measure_memory(self.arthur_items) / (<NUM_LIT> * <NUM_LIT>)<EOL><DEDENT>except RuntimeError as ex:<EOL><INDENT>logger.warning("<STR_LIT>", ex)<EOL>memory_size = self.ARTHUR_LAST_MEMORY_SIZE<EOL><DEDENT>self.ARTHUR_LAST_MEMORY_CHECK_TIME = time.time() - self.ARTHUR_LAST_MEMORY_CHECK<EOL>logger.debug("<STR_LIT>",<EOL>memory_size, self.ARTHUR_LAST_MEMORY_CHECK_TIME)<EOL>self.ARTHUR_LAST_MEMORY_SIZE = memory_size<EOL><DEDENT>if self.ARTHUR_LAST_MEMORY_SIZE > self.ARTHUR_MAX_MEMORY_SIZE:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>db_url = self.config.get_conf()['<STR_LIT>']['<STR_LIT>']<EOL>conn = redis.StrictRedis.from_url(db_url)<EOL>logger.debug("<STR_LIT>", db_url)<EOL>pipe = conn.pipeline()<EOL>pipe.lrange(Q_STORAGE_ITEMS, <NUM_LIT:0>, self.ARTHUR_REDIS_ITEMS - <NUM_LIT:1>)<EOL>pipe.ltrim(Q_STORAGE_ITEMS, self.ARTHUR_REDIS_ITEMS, -<NUM_LIT:1>)<EOL>items = pipe.execute()[<NUM_LIT:0>]<EOL>for item in items:<EOL><INDENT>arthur_item = pickle.loads(item)<EOL>if arthur_item['<STR_LIT>'] not in self.arthur_items:<EOL><INDENT>self.arthur_items[arthur_item['<STR_LIT>']] = []<EOL><DEDENT>self.arthur_items[arthur_item['<STR_LIT>']].append(arthur_item)<EOL><DEDENT>for tag in self.arthur_items:<EOL><INDENT>if self.arthur_items[tag]:<EOL><INDENT>logger.debug("<STR_LIT>", tag, len(self.arthur_items[tag]))<EOL><DEDENT><DEDENT><DEDENT>

Feed Ocean with backend data collected from arthur redis queue


<EOL>self.__feed_arthur()<EOL>tag = self.backend_tag(repo)<EOL>logger.debug("<STR_LIT>", self.arthur_items.keys())<EOL>logger.debug("<STR_LIT>", tag)<EOL>if tag in self.arthur_items:<EOL><INDENT>logger.debug("<STR_LIT>", tag)<EOL>while self.arthur_items[tag]:<EOL><INDENT>yield self.arthur_items[tag].pop()<EOL><DEDENT><DEDENT>


if uri.split('<STR_LIT:/>')[<NUM_LIT:2>] == '<STR_LIT>':<EOL><INDENT>return True<EOL><DEDENT>elif uri.split('<STR_LIT:/>')[<NUM_LIT:2>] == '<STR_LIT>':<EOL><INDENT>if uri.split('<STR_LIT:/>')[<NUM_LIT:5>] == '<STR_LIT>':<EOL><INDENT>return True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise GithubFileNotFound('<STR_LIT>' % uri)<EOL><DEDENT>

Checks the URI looks like a RAW uri in github:

        - 'https://raw.githubusercontent.com/github/hubot/master/README.md'
        - 'https://github.com/github/hubot/raw/master/README.md'

        :param uri: uri of the file


logger.debug("<STR_LIT>" % (uri))<EOL>self.__check_looks_like_uri(uri)<EOL>try:<EOL><INDENT>req = urllib.request.Request(uri)<EOL>req.add_header('<STR_LIT>', '<STR_LIT>' % self.token)<EOL>r = urllib.request.urlopen(req)<EOL><DEDENT>except urllib.error.HTTPError as err:<EOL><INDENT>if err.code == <NUM_LIT>:<EOL><INDENT>raise GithubFileNotFound('<STR_LIT>' % uri)<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return r.read().decode("<STR_LIT:utf-8>")<EOL>

Reads the file from Github

        :param uri: URI of the Github raw File

        :returns: UTF-8 text with the content


self.execute_batch_tasks(tasks_cls)<EOL>

Just a wrapper to the execute_batch_tasks method


if self.conf['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>tasks_cls = [TaskPanels, TaskPanelsMenu]<EOL>self.execute_tasks(tasks_cls)<EOL><DEDENT>if self.conf['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>tasks_cls = [TaskInitSortingHat]<EOL>self.execute_tasks(tasks_cls)<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>tasks_cls = [TaskProjects]<EOL>self.execute_tasks(tasks_cls)<EOL>logger.info("<STR_LIT>")<EOL>return<EOL>


def execute_batch_tasks(self, tasks_cls, big_delay=<NUM_LIT:0>, small_delay=<NUM_LIT:0>, wait_for_threads=True):

def _split_tasks(tasks_cls):<EOL><INDENT>"""<STR_LIT>"""<EOL>backend_t = []<EOL>global_t = []<EOL>for t in tasks_cls:<EOL><INDENT>if t.is_backend_task(t):<EOL><INDENT>backend_t.append(t)<EOL><DEDENT>else:<EOL><INDENT>global_t.append(t)<EOL><DEDENT><DEDENT>return backend_t, global_t<EOL><DEDENT>backend_tasks, global_tasks = _split_tasks(tasks_cls)<EOL>logger.debug('<STR_LIT>' % (backend_tasks))<EOL>logger.debug('<STR_LIT>' % (global_tasks))<EOL>threads = []<EOL>stopper = threading.Event()<EOL>if len(backend_tasks) > <NUM_LIT:0>:<EOL><INDENT>repos_backend = self._get_repos_by_backend()<EOL>for backend in repos_backend:<EOL><INDENT>t = TasksManager(backend_tasks, backend, stopper, self.config, small_delay)<EOL>threads.append(t)<EOL>t.start()<EOL><DEDENT><DEDENT>if len(global_tasks) > <NUM_LIT:0>:<EOL><INDENT>gt = TasksManager(global_tasks, "<STR_LIT>", stopper, self.config, big_delay)<EOL>threads.append(gt)<EOL>gt.start()<EOL>if big_delay > <NUM_LIT:0>:<EOL><INDENT>when = datetime.now() + timedelta(seconds=big_delay)<EOL>when_str = when.strftime('<STR_LIT>')<EOL>logger.info("<STR_LIT>" % (global_tasks, when_str))<EOL><DEDENT><DEDENT>if wait_for_threads:<EOL><INDENT>time.sleep(<NUM_LIT:1>)  <EOL>stopper.set()  <EOL><DEDENT>for t in threads:<EOL><INDENT>t.join()<EOL><DEDENT>self.__check_queue_for_errors()<EOL>logger.debug("<STR_LIT>")<EOL>

Start a task manager per backend to complete the tasks.

:param task_cls: list of tasks classes to be executed
:param big_delay: seconds before global tasks are executed, should be days usually
:param small_delay: seconds before backend tasks are executed, should be minutes
:param wait_for_threads: boolean to set when threads are infinite or
                        should be synchronized in a meeting point


self.execute_batch_tasks(tasks_cls,<EOL>self.conf['<STR_LIT>']['<STR_LIT>'],<EOL>self.conf['<STR_LIT>']['<STR_LIT>'], False)<EOL>


This method defines the workflow of SirMordred. So it calls to:
- initialize the databases
- execute the different phases for the first iteration
  (collection, identities, enrichment)
- start the collection and enrichment in parallel by data source
- start also the Sorting Hat merge


mbox_archives = '<STR_LIT>'<EOL>mailing_lists_projects = [project for project in projects if '<STR_LIT>' in projects[project]]<EOL>for mailing_lists in mailing_lists_projects:<EOL><INDENT>projects[mailing_lists]['<STR_LIT>'] = []<EOL>for mailing_list in projects[mailing_lists]['<STR_LIT>']:<EOL><INDENT>if '<STR_LIT>' in mailing_list:<EOL><INDENT>name = mailing_list.split('<STR_LIT>')[<NUM_LIT:1>]<EOL><DEDENT>elif '<STR_LIT>' in mailing_list:<EOL><INDENT>name = mailing_list.split('<STR_LIT>')[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>name = mailing_list.split('<STR_LIT:@>')[<NUM_LIT:0>]<EOL><DEDENT>list_new = "<STR_LIT>" % (name, mbox_archives, name, name)<EOL>projects[mailing_lists]['<STR_LIT>'].append(list_new)<EOL><DEDENT><DEDENT>return projects<EOL>

Compose projects.json only for mbox, but using the mailing_lists lists

    change: 'https://dev.eclipse.org/mailman/listinfo/emft-dev'
    to: 'emfg-dev /home/bitergia/mboxes/emft-dev.mbox/emft-dev.mbox

    :param projects: projects.json
    :return: projects.json with mbox


projects = compose_git(projects, data)<EOL>projects = compose_mailing_lists(projects, data)<EOL>projects = compose_bugzilla(projects, data)<EOL>projects = compose_github(projects, data)<EOL>projects = compose_gerrit(projects)<EOL>projects = compose_mbox(projects)<EOL>return projects<EOL>

Compose projects.json with all data sources

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with all data sources


for p in [project for project in data if len(data[project]['<STR_LIT>']) > <NUM_LIT:0>]:<EOL><INDENT>repos = []<EOL>for url in data[p]['<STR_LIT>']:<EOL><INDENT>if len(url['<STR_LIT:url>'].split()) > <NUM_LIT:1>:  <EOL><INDENT>repo = url['<STR_LIT:url>'].split()[<NUM_LIT:1>].replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>repo = url['<STR_LIT:url>'].replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>if repo not in repos:<EOL><INDENT>repos.append(repo)<EOL><DEDENT><DEDENT>projects[p]['<STR_LIT>'] = repos<EOL><DEDENT>return projects<EOL>

Compose projects.json for git

    We need to replace '/c/' by '/gitroot/' for instance

    change: 'http://git.eclipse.org/c/xwt/org.eclipse.xwt.git'
    to: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with git


@classmethod<EOL><INDENT>def get_repos_by_backend_section(cls, backend_section, raw=True):<DEDENT>

repos = []<EOL>projects = TaskProjects.get_projects()<EOL>for pro in projects:<EOL><INDENT>if backend_section in projects[pro]:<EOL><INDENT>if cls.GLOBAL_PROJECT not in projects:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT>else:<EOL><INDENT>if raw:<EOL><INDENT>if pro != cls.GLOBAL_PROJECT:<EOL><INDENT>if backend_section not in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT>elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[cls.GLOBAL_PROJECT][backend_section]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>not_in_unknown = [projects[pro] for pro in projects if pro != cls.GLOBAL_PROJECT][<NUM_LIT:0>]<EOL>if backend_section not in not_in_unknown:<EOL><INDENT>repos += projects[cls.GLOBAL_PROJECT][backend_section]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if pro != cls.GLOBAL_PROJECT:<EOL><INDENT>if backend_section not in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT>elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>not_in_unknown_prj = [projects[prj] for prj in projects if prj != cls.GLOBAL_PROJECT]<EOL>not_in_unknown_sections = list(set([section for prj in not_in_unknown_prj<EOL>for section in list(prj.keys())]))<EOL>if backend_section not in not_in_unknown_sections:<EOL><INDENT>repos += projects[pro][backend_section]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>logger.debug("<STR_LIT>", backend_section, repos, raw)<EOL>repos = list(set(repos))<EOL>return repos<EOL>

return list with the repositories for a backend_section


def __init__(self, tasks_cls, backend_section, stopper, config, timer=<NUM_LIT:0>):

super().__init__(name=backend_section)  <EOL>self.config = config<EOL>self.tasks_cls = tasks_cls  <EOL>self.tasks = []  <EOL>self.backend_section = backend_section<EOL>self.stopper = stopper  <EOL>self.timer = timer<EOL>self.thread_id = None<EOL>

:tasks_cls : tasks classes to be executed using the backend
:backend_section: perceval backend section name
:config: config object for the manager


version = None<EOL>es_url = self.conf['<STR_LIT>']['<STR_LIT:url>']<EOL>config_url = '<STR_LIT>'<EOL>url = urijoin(es_url, config_url)<EOL>version = None<EOL>try:<EOL><INDENT>res = self.grimoire_con.get(url)<EOL>res.raise_for_status()<EOL>version = res.json()['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>']<EOL>logger.debug("<STR_LIT>", version)<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT>return version<EOL>

Get the kibiter vesion.

        :param major: major Elasticsearch version


if kibiter_major == "<STR_LIT>":<EOL><INDENT>resource = "<STR_LIT>"<EOL>data = {"<STR_LIT>": {"<STR_LIT:name>": self.project_name}}<EOL>mapping_resource = "<STR_LIT>"<EOL>mapping = {"<STR_LIT>": "<STR_LIT:true>"}<EOL>url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'], resource)<EOL>mapping_url = urijoin(self.conf['<STR_LIT>']['<STR_LIT:url>'],<EOL>mapping_resource)<EOL>logger.debug("<STR_LIT>")<EOL>res = self.grimoire_con.put(mapping_url, data=json.dumps(mapping),<EOL>headers=ES6_HEADER)<EOL>try:<EOL><INDENT>res.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>logger.error(res.json())<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>res = self.grimoire_con.post(url, data=json.dumps(data),<EOL>headers=ES6_HEADER)<EOL>try:<EOL><INDENT>res.raise_for_status()<EOL><DEDENT>except requests.exceptions.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>logger.error(res.json())<EOL><DEDENT><DEDENT>

Upload to Kibiter the title for the dashboard.

        The title is shown on top of the dashboard menu, and is Usually
        the name of the project being dashboarded.
        This is done only for Kibiter 6.x.

        :param kibiter_major: major version of kibiter


menu_entries = []<EOL>for entry in self.panels_menu:<EOL><INDENT>if entry['<STR_LIT:source>'] not in self.data_sources:<EOL><INDENT>continue<EOL><DEDENT>parent_menu_item = {<EOL>'<STR_LIT:name>': entry['<STR_LIT:name>'],<EOL>'<STR_LIT:title>': entry['<STR_LIT:name>'],<EOL>'<STR_LIT:description>': "<STR_LIT>",<EOL>'<STR_LIT:type>': "<STR_LIT>",<EOL>'<STR_LIT>': []<EOL>}<EOL>for subentry in entry['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>dash_name = get_dashboard_name(subentry['<STR_LIT>'])<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>logging.error("<STR_LIT>", subentry['<STR_LIT>'])<EOL>continue<EOL><DEDENT>child_item = {<EOL>"<STR_LIT:name>": subentry['<STR_LIT:name>'],<EOL>"<STR_LIT:title>": subentry['<STR_LIT:name>'],<EOL>"<STR_LIT:description>": "<STR_LIT>",<EOL>"<STR_LIT:type>": "<STR_LIT>",<EOL>"<STR_LIT>": dash_name<EOL>}<EOL>parent_menu_item['<STR_LIT>'].append(child_item)<EOL><DEDENT>menu_entries.append(parent_menu_item)<EOL><DEDENT>return menu_entries<EOL>

Get the menu entries from the panel definition


enrich_es = self.conf['<STR_LIT>']['<STR_LIT:url>']<EOL>sortinghat_db = self.db<EOL>current_data_source = self.get_backend(self.backend_section)<EOL>active_data_sources = self.config.get_active_data_sources()<EOL>if retention_time is None:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>if retention_time <= <NUM_LIT:0>:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>retain_identities(retention_time, enrich_es, sortinghat_db, current_data_source, active_data_sources)<EOL>

Retain the identities in SortingHat based on the `retention_time`
        value declared in the setup.cfg.

        :param retention_time: maximum number of minutes wrt the current date to retain the SortingHat data


cfg = self.config.get_conf()<EOL>if '<STR_LIT>' not in cfg[self.backend_section] or notcfg[self.backend_section]['<STR_LIT>']:<EOL><INDENT>logger.debug('<STR_LIT>' % self.backend_section)<EOL>return<EOL><DEDENT>studies = [study for study in cfg[self.backend_section]['<STR_LIT>'] if study.strip() != "<STR_LIT>"]<EOL>if not studies:<EOL><INDENT>logger.debug('<STR_LIT>' % self.backend_section)<EOL>return<EOL><DEDENT>logger.debug("<STR_LIT>" % (self.backend_section, studies))<EOL>time.sleep(<NUM_LIT:2>)  <EOL>enrich_backend = self._get_enrich_backend()<EOL>ocean_backend = self._get_ocean_backend(enrich_backend)<EOL>active_studies = []<EOL>all_studies = enrich_backend.studies<EOL>all_studies_names = [study.__name__ for study in enrich_backend.studies]<EOL>logger.debug("<STR_LIT>", self.backend_section, all_studies_names)<EOL>logger.debug("<STR_LIT>", studies)<EOL>cfg_studies_types = [study.split("<STR_LIT::>")[<NUM_LIT:0>] for study in studies]<EOL>if not set(cfg_studies_types).issubset(set(all_studies_names)):<EOL><INDENT>logger.error('<STR_LIT>', self.backend_section, studies)<EOL>raise RuntimeError('<STR_LIT>', self.backend_section, studies)<EOL><DEDENT>for study in enrich_backend.studies:<EOL><INDENT>if study.__name__ in cfg_studies_types:<EOL><INDENT>active_studies.append(study)<EOL><DEDENT><DEDENT>enrich_backend.studies = active_studies<EOL>print("<STR_LIT>" % (self.backend_section,<EOL>[study for study in studies]))<EOL>studies_args = self.__load_studies()<EOL>do_studies(ocean_backend, enrich_backend, studies_args, retention_time=retention_time)<EOL>enrich_backend.studies = all_studies<EOL>

Execute the studies configured for the current backend


if '<STR_LIT>' not in self.conf[self.backend_section] or'<STR_LIT>' not in self.conf[self.backend_section]['<STR_LIT>']:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>aoc_index = self.conf['<STR_LIT>'].get('<STR_LIT>', GitEnrich.GIT_AOC_ENRICHED)<EOL>if not aoc_index:<EOL><INDENT>aoc_index = GitEnrich.GIT_AOC_ENRICHED<EOL><DEDENT>logger.debug("<STR_LIT>", aoc_index)<EOL>es = Elasticsearch([self.conf['<STR_LIT>']['<STR_LIT:url>']], timeout=<NUM_LIT:100>,<EOL>verify_certs=self._get_enrich_backend().elastic.requests.verify)<EOL>if not es.indices.exists(index=aoc_index):<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>aoc_backend = GitEnrich(self.db_sh, None, cfg['<STR_LIT>']['<STR_LIT>'],<EOL>self.db_user, self.db_password, self.db_host)<EOL>aoc_backend.mapping = None<EOL>aoc_backend.roles = ['<STR_LIT>']<EOL>elastic_enrich = get_elastic(self.conf['<STR_LIT>']['<STR_LIT:url>'],<EOL>aoc_index, clean=False, backend=aoc_backend)<EOL>aoc_backend.set_elastic(elastic_enrich)<EOL>self.__autorefresh(aoc_backend, studies=True)<EOL>

Execute autorefresh for areas of code study if configured


task = TaskPanels(config)<EOL>task.execute()<EOL>task = TaskPanelsMenu(config)<EOL>task.execute()<EOL>logging.info("<STR_LIT>")<EOL>

Execute the panels phase

    :param config: a Mordred config object


TaskProjects(config).execute()<EOL>task = TaskIdentitiesMerge(config)<EOL>task.execute()<EOL>logging.info("<STR_LIT>")<EOL>

Execute the merge identities phase

    :param config: a Mordred config object


parser = argparse.ArgumentParser(add_help=False)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>help=argparse.SUPPRESS)<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", dest='<STR_LIT>', default=[],<EOL>nargs='<STR_LIT:*>', help="<STR_LIT>")<EOL>if len(sys.argv) == <NUM_LIT:1>:<EOL><INDENT>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return parser<EOL>


TaskProjects(config).execute()<EOL>task = TaskEnrich(config, backend_section=backend_section)<EOL>try:<EOL><INDENT>task.execute()<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>except Exception as e:<EOL><INDENT>logging.error(str(e))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>

Execute the enrich phase for a given backend section

    :param config: a Mordred config object
    :param backend_section: the backend section where the enrich phase is executed


return self._color[<NUM_LIT:2>]<EOL>

The blue component of the RGB color representation.


if isinstance(other, Color):<EOL><INDENT>return self.equality_fn(self, other)<EOL><DEDENT>return False<EOL>


return rgb_to_web(offset_random_rgb(seed, amount))<EOL>

Given a seed color, generate a specified number of random colors (1 color by default) determined by a randomized
offset from the seed.

:param seed:
:param amount:
:return:


return not self.__eq__(other)<EOL>


return rgb_to_web(random_rgb())<EOL>

Generate a uniformly random WEB value.

:return:


return self._color[<NUM_LIT:1>]<EOL>

The green component of the RGB color representation.


def color_run(start_color, end_color, step_count, inclusive=True, to_color=True):

if isinstance(start_color, Color):<EOL><INDENT>start_color = start_color.rgb<EOL><DEDENT>if isinstance(end_color, Color):<EOL><INDENT>end_color = end_color.rgb<EOL><DEDENT>step = tuple((end_color[i] - start_color[i])/step_count for i in range(<NUM_LIT:3>))<EOL>add = lambda x, y: tuple(sum(z) for z in zip(x, y))<EOL>mult = lambda x, y: tuple(y * z for z in x)<EOL>run = [add(start_color, mult(step, i)) for i in range(<NUM_LIT:1>, step_count)]<EOL>if inclusive:<EOL><INDENT>run = [start_color] + run + [end_color]<EOL><DEDENT>return run if not to_color else [Color(c) for c in run]<EOL>

Given a start color, end color, and a number of steps, returns a list of colors which represent a 'scale' between
the start and end color.

:param start_color: The color starting the run
:param end_color: The color ending the run
:param step_count: The number of colors to have between the start and end color
:param inclusive: Flag determining whether to include start and end values in run (default True)
:param to_color: Flag indicating return values should be Color objects (default True)
:return: List of colors between the start and end color
:rtype: list


return rgb_to_hex(random_rgb())<EOL>

Generate a uniformly random HEX value.

:return: A string representing a random HEX value between 000000 and FFFFFF inclusive


r, g, b = seed<EOL>results = []<EOL>for _ in range(amount):<EOL><INDENT>base_val = ((r + g + b) / <NUM_LIT:3>) + <NUM_LIT:1>  <EOL>new_val = base_val + (random.random() * rgb_max_val / <NUM_LIT:5>)  <EOL>ratio = new_val / base_val<EOL>results.append((min(int(r*ratio), rgb_max_val), min(int(g*ratio), rgb_max_val), min(int(b*ratio), rgb_max_val)))<EOL><DEDENT>return results[<NUM_LIT:0>] if len(results) > <NUM_LIT:1> else results<EOL>


def __init__(self, color=None, **kwargs):

self.equality_fn = RGB_eq<EOL>self.arithmetic = ArithmeticModel.LIGHT<EOL>if isinstance(color, Color):<EOL><INDENT>self._color = color._color<EOL><DEDENT>else:<EOL><INDENT>self._color = color if color else rgb_min<EOL><DEDENT>for k, v in kwargs.items():<EOL><INDENT>setattr(self, k, v)<EOL><DEDENT>


return self._color<EOL>


return "<STR_LIT:{}>".format(self._color)<EOL>


return self._color[<NUM_LIT:0>]<EOL>

The red component of the RGB color representation.


return rgb_to_web(self.rgb)<EOL>


return rgb_to_hex(self.rgb)<EOL>

A 6-char HEX representation of the color, with a prepended octothorpe.


try:<EOL><INDENT>return web_colors[hex_to_rgb(_hex)]<EOL><DEDENT>except KeyError:<EOL><INDENT>return _hex<EOL><DEDENT>

Convert a HEX color representation to a WEB color representation.

hex :: hex -> [000000, FFFFFF]

:param _hex: The 3- or 6-char hexadecimal string representing the color value.
:return: WEB representation of the input HEX value.
:rtype: str


return rgb_to_web(hsv_to_rgb(hsv))<EOL>

Convert an HSV color representation to a WEB color representation.

(h, s, v) :: h -> [0, 360)
             s -> [0, 1]
             v -> [0, 1]

:param hsv: A tuple of three numeric values corresponding to the hue, saturation, and value.
:return: WEB representation of the input HSV value.
:rtype: str


return rgb_to_web(yiq_to_rgb(yiq))<EOL>

Convert a YIQ color representation to a WEB color representation.

(y, i, q) :: y -> [0, 1]
             i -> [-0.5957, 0.5957]
             q -> [-0.5226, 0.5226]

:param yiq: A tuple of three numeric values corresponding to the luma and chrominance.
:return: WEB representation of the input YIQ value.
:rtype: str


return rgb_to_yiq(hex_to_rgb(_hex))<EOL>

Convert a HEX color representation to a YIQ color representation.

hex :: hex -> [000000, FFFFFF]

:param _hex: The 3- or 6-char hexadecimal string representing the color value.
:return: YIQ representation of the input HEX value.
:rtype: tuple


return rgb_to_yiq(hsv_to_rgb(hsv))<EOL>

Convert an HSV color representation to a YIQ color representation.

(h, s, v) :: h -> [0, 360)
             s -> [0, 1]
             v -> [0, 1]

:param hsv: A tuple of three numeric values corresponding to the hue, saturation, and value.
:return: YIQ representation of the input HSV value.
:rtype: tuple


return rgb_to_yiq(web_to_rgb(web))<EOL>

Convert a WEB color representation to a YIQ color representation.

web :: web -> [000000, FFFFFF]
            | in static.web_colors

:param web: The WEB string representation of a color.
:return: YIQ representation of the input WEB value.
:rtype: tuple


self.base_dict = {}<EOL>_start_arrays = []<EOL>for tokens, word in self.chain_generator():<EOL><INDENT>self.base_dict.setdefault(tokens, []).append(word)<EOL>if tokens[<NUM_LIT:0>] == "<STR_LIT>":  <EOL><INDENT>_start_arrays.append(tokens)<EOL><DEDENT><DEDENT>self.start_arrays = tuple(<EOL>frozenset(self.get_corrected_arrays(_start_arrays))<EOL>)<EOL>

Метод создаёт базовый словарь, на основе массива токенов.
Вызывается из метода обновления.


def get_optimal_variant(self, variants, start_words, **kwargs):

if not start_words:<EOL><INDENT>return (choice(variants), {})<EOL><DEDENT>_variants = []<EOL>_weights = []<EOL>for tok in frozenset(variants):<EOL><INDENT>if not self.token_is_correct(tok):<EOL><INDENT>continue<EOL><DEDENT>weight = variants.count(tok)<EOL>for word in start_words:<EOL><INDENT>for token in self.ONLY_WORDS.finditer(word.strip().lower()):<EOL><INDENT>if token.group() == tok:<EOL><INDENT>weight <<= <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>_variants.append(tok)<EOL>_weights.append(weight)<EOL><DEDENT>if not _variants:<EOL><INDENT>return (choice(variants), {})<EOL><DEDENT>return (choices(_variants, weights=_weights, k=<NUM_LIT:1>)[<NUM_LIT:0>], {})<EOL>

Возвращает оптимальный вариант, из выборки.


def start_generation(self, *start_words, **kwargs):

out_text = "<STR_LIT>"<EOL>_need_capialize = True<EOL>for token in self._get_generate_tokens(*start_words, **kwargs):<EOL><INDENT>if token in "<STR_LIT>":<EOL><INDENT>_need_capialize = True<EOL>continue<EOL><DEDENT>if self.ONLY_WORDS.search(token):<EOL><INDENT>out_text += "<STR_LIT:U+0020>"<EOL><DEDENT>if _need_capialize:<EOL><INDENT>_need_capialize = False<EOL>token = token.title()<EOL><DEDENT>out_text += token<EOL><DEDENT>return out_text.strip()<EOL>

Генерирует предложение.
:start_words: Попытаться начать предложение с этих слов.


if not isinstance(text, str):<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>")<EOL><DEDENT>text = text.strip().lower()<EOL>need_start_token = True<EOL>token = "<STR_LIT:$>"  <EOL>for token in self.WORD_OR_MARKS.finditer(text):<EOL><INDENT>token = token.group()<EOL>if need_start_token:<EOL><INDENT>need_start_token = False<EOL>yield "<STR_LIT>"<EOL><DEDENT>yield token<EOL>if self.END_TOKENS.search(token):<EOL><INDENT>need_start_token = True<EOL>yield "<STR_LIT:$>"<EOL><DEDENT><DEDENT>if token != "<STR_LIT:$>":<EOL><INDENT>yield "<STR_LIT:$>"<EOL><DEDENT>

Возвращает генератор токенов, из текста.


name = name or "<STR_LIT>"<EOL>dump_file = os_join(<EOL>self.temp_folder,<EOL>"<STR_LIT>".format(name)<EOL>)<EOL>if not isfile(dump_file):<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>".format(dump_file))<EOL><DEDENT>with open(dump_file, "<STR_LIT:rb>") as js_file:<EOL><INDENT>self.tokens_array = tuple(json.load(js_file))<EOL><DEDENT>self.create_base()<EOL>

Загружает базу с жёсткого диска.
Текущая база заменяется.
:name: Имя файла, без расширения.


file_path = abspath(file_path)<EOL>if not isfile(file_path):<EOL><INDENT>raise MarkovTextExcept("<STR_LIT>")<EOL><DEDENT>with open(file_path, "<STR_LIT:rb>") as txt_file:<EOL><INDENT>for line in txt_file:<EOL><INDENT>text = line.decode("<STR_LIT:utf-8>", "<STR_LIT:ignore>").strip()<EOL>if not text:<EOL><INDENT>continue<EOL><DEDENT>yield from self._parse_from_text(text)<EOL><DEDENT><DEDENT>

см. описание _parse_from_text.
Только на вход подаётся не текст, а путь к файлу.


tokens = []<EOL>class Vars:<EOL><INDENT>start_pos = -<NUM_LIT:1><EOL>last_type = '<STR_LIT:o>'<EOL><DEDENT>def update(c, i):<EOL><INDENT>if c.isalpha() or c in '<STR_LIT>':<EOL><INDENT>t = '<STR_LIT:a>'<EOL><DEDENT>elif c.isdigit() or c == '<STR_LIT:#>':<EOL><INDENT>t = '<STR_LIT:n>'<EOL><DEDENT>elif c.isspace():<EOL><INDENT>t = '<STR_LIT:s>'<EOL><DEDENT>else:<EOL><INDENT>t = '<STR_LIT:o>'<EOL><DEDENT>if t != Vars.last_type or t == '<STR_LIT:o>':<EOL><INDENT>if Vars.start_pos >= <NUM_LIT:0>:<EOL><INDENT>token = sentence[Vars.start_pos:i].lower()<EOL>if token not in '<STR_LIT>':<EOL><INDENT>tokens.append(token)<EOL><DEDENT><DEDENT>Vars.start_pos = -<NUM_LIT:1> if t == '<STR_LIT:s>' else i<EOL><DEDENT>Vars.last_type = t<EOL><DEDENT>for i, char in enumerate(sentence):<EOL><INDENT>update(char, i)<EOL><DEDENT>update('<STR_LIT:U+0020>', len(sentence))<EOL>return tokens<EOL>

Converts a single sentence into a list of individual significant units
Args:
    sentence (str): Input string ie. 'This is a sentence.'
Returns:
    list<str>: List of tokens ie. ['this', 'is', 'a', 'sentence']


return SentenceTreeParser(sent).expand_parentheses()<EOL>

['1', '(', '2', '|', '3, ')'] -> [['1', '2'], ['1', '3']]
For example:

Will it (rain|pour) (today|tomorrow|)?

---->

Will it rain today?
Will it rain tomorrow?
Will it rain?
Will it pour today?
Will it pour tomorrow?
Will it pour?

Args:
    sent (list<str>): List of tokens in sentence
Returns:
    list<list<str>>: Multiple possible sentences from original


data = {}<EOL>for inp, out in zip(inputs, outputs):<EOL><INDENT>tup = tuple(inp)<EOL>if tup in data:<EOL><INDENT>data[tup].append(out)<EOL><DEDENT>else:<EOL><INDENT>data[tup] = [out]<EOL><DEDENT><DEDENT>inputs, outputs = [], []<EOL>for inp, outs in data.items():<EOL><INDENT>inputs.append(list(inp))<EOL>combined = [<NUM_LIT:0>] * len(outs[<NUM_LIT:0>])<EOL>for i in range(len(combined)):<EOL><INDENT>combined[i] = max(j[i] for j in outs)<EOL><DEDENT>outputs.append(combined)<EOL><DEDENT>return inputs, outputs<EOL>

Checks for duplicate inputs and if there are any,
remove one and set the output to the max of the two outputs
Args:
    inputs (list<list<float>>): Array of input vectors
    outputs (list<list<float>>): Array of output vectors
Returns:
    tuple<inputs, outputs>: The modified inputs and outputs


return tree.expand()<EOL>

Expand a list of sub sentences to all combinated sentences.
['1', ['2', '3']] -> [['1', '2'], ['1', '3']]


old_expanded = [[]]<EOL>for sub in self._tree:<EOL><INDENT>sub_expanded = sub.expand()<EOL>new_expanded = []<EOL>while len(old_expanded) > <NUM_LIT:0>:<EOL><INDENT>sentence = old_expanded.pop()<EOL>for new in sub_expanded:<EOL><INDENT>new_expanded.append(sentence + new)<EOL><DEDENT><DEDENT>old_expanded = new_expanded<EOL><DEDENT>return old_expanded<EOL>

Creates a combination of all sub-sentences.

Returns:
    List<List<str>>: A list with all subsentence expansions combined in
                        every possible way


<EOL>sentence_list = []<EOL>cur_sentence = []<EOL>sentence_list.append(Sentence(cur_sentence))<EOL>while self._current_position < len(self.tokens):<EOL><INDENT>cur = self.tokens[self._current_position]<EOL>self._current_position += <NUM_LIT:1><EOL>if cur == '<STR_LIT:(>':<EOL><INDENT>subexpr = self._parse_expr()<EOL>normal_brackets = False<EOL>if len(subexpr.tree()) == <NUM_LIT:1>:<EOL><INDENT>normal_brackets = True<EOL>cur_sentence.append(Word('<STR_LIT:(>'))<EOL><DEDENT>cur_sentence.append(subexpr)<EOL>if normal_brackets:<EOL><INDENT>cur_sentence.append(Word('<STR_LIT:)>'))<EOL><DEDENT><DEDENT>elif cur == '<STR_LIT:|>':<EOL><INDENT>cur_sentence = []<EOL>sentence_list.append(Sentence(cur_sentence))<EOL><DEDENT>elif cur == '<STR_LIT:)>':<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>cur_sentence.append(Word(cur))<EOL><DEDENT><DEDENT>return Options(sentence_list)<EOL>

Generate sentence token trees from the current position to
the next closing parentheses / end of the list and return it
['1', '(', '2', '|', '3, ')'] -> ['1', [['2'], ['3']]]
['2', '|', '3'] -> [['2'], ['3']]


return [[self._tree]]<EOL>

Creates one sentence that contains exactly that word.

Returns:
    List<List<str>>: A list with the given string as sentence
                        (= token/string list)


return [[]]<EOL>

Expanded version of the fragment. In this case an empty sentence.

Returns:
    List<List<str>>: A list with an empty sentence (= token/string list)


@_save_args<EOL><INDENT>def add_entity(self, name, lines, reload_cache=False):<DEDENT>

Entity.verify_name(name)<EOL>self.entities.add(Entity.wrap_name(name), lines, reload_cache)<EOL>self.padaos.add_entity(name, lines)<EOL>self.must_train = True<EOL>

Adds an entity that matches the given lines.

Example:
    self.add_intent('weather', ['will it rain on {weekday}?'])
    self.add_entity('{weekday}', ['monday', 'tuesday', 'wednesday'])  # ...

Args:
    name (str): The name of the entity
    lines (list<str>): Lines of example extracted entities
    reload_cache (bool): Whether to refresh all of cache


def train(self, debug=True, force=False, single_thread=False, timeout=<NUM_LIT:20>):

if not self.must_train and not force:<EOL><INDENT>return<EOL><DEDENT>self.padaos.compile()<EOL>self.train_thread = Thread(target=self._train, kwargs=dict(<EOL>debug=debug,<EOL>single_thread=single_thread,<EOL>timeout=timeout<EOL>), daemon=True)<EOL>self.train_thread.start()<EOL>self.train_thread.join(timeout)<EOL>self.must_train = False<EOL>return not self.train_thread.is_alive()<EOL>

Trains all the loaded intents that need to be updated
If a cache file exists with the same hash as the intent file,
the intent will not be trained and just loaded from file

Args:
    debug (bool): Whether to print a message to stdout each time a new intent is trained
    force (bool): Whether to force training if already finished
    single_thread (bool): Whether to force running in a single thread
    timeout (float): Seconds before cancelling training
Returns:
    bool: True if training succeeded without timeout


matches = self.calc_intents(query)<EOL>if len(matches) == <NUM_LIT:0>:<EOL><INDENT>return MatchData('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>best_match = max(matches, key=lambda x: x.conf)<EOL>best_matches = (match for match in matches if match.conf == best_match.conf)<EOL>return min(best_matches, key=lambda x: sum(map(len, x.matches.values())))<EOL>

Tests all the intents against the query and returns
match data of the best intent

Args:
    query (str): Input sentence to test against intents
Returns:
    MatchData: Best intent match


@_save_args<EOL><INDENT>def load_file(self, *args, **kwargs):<DEDENT>

self.load_intent(*args, **kwargs)<EOL>


@_save_args<EOL><INDENT>def add_intent(self, name, lines, reload_cache=False):<DEDENT>

self.intents.add(name, lines, reload_cache)<EOL>self.padaos.add_intent(name, lines)<EOL>self.must_train = True<EOL>

Creates a new intent, optionally checking the cache first

Args:
    name (str): The associated name of the intent
    lines (list<str>): All the sentences that should activate the intent
    reload_cache: Whether to ignore cached intent if exists


if self.must_train:<EOL><INDENT>self.train()<EOL><DEDENT>intents = {} if self.train_thread and self.train_thread.is_alive() else {<EOL>i.name: i for i in self.intents.calc_intents(query, self.entities)<EOL>}<EOL>sent = tokenize(query)<EOL>for perfect_match in self.padaos.calc_intents(query):<EOL><INDENT>name = perfect_match['<STR_LIT:name>']<EOL>intents[name] = MatchData(name, sent, matches=perfect_match['<STR_LIT>'], conf=<NUM_LIT:1.0>)<EOL><DEDENT>return list(intents.values())<EOL>

Tests all the intents against the query and returns
data on how well each one matched against the query

Args:
    query (str): Input sentence to test against intents
Returns:
    list<MatchData>: List of intent matches
See calc_intent() for a description of the returned MatchData


@_save_args<EOL><INDENT>def load_intent(self, name, file_name, reload_cache=False):<DEDENT>

self.intents.load(name, file_name, reload_cache)<EOL>with open(file_name) as f:<EOL><INDENT>self.padaos.add_intent(name, f.read().split('<STR_LIT:\n>'))<EOL><DEDENT>self.must_train = True<EOL>

Loads an intent, optionally checking the cache first

Args:
    name (str): The associated name of the intent
    file_name (str): The location of the intent file
    reload_cache (bool): Whether to refresh all of cache


@_save_args<EOL><INDENT>def load_entity(self, name, file_name, reload_cache=False):<DEDENT>

Entity.verify_name(name)<EOL>self.entities.load(Entity.wrap_name(name), file_name, reload_cache)<EOL>with open(file_name) as f:<EOL><INDENT>self.padaos.add_entity(name, f.read().split('<STR_LIT:\n>'))<EOL><DEDENT>self.must_train = True<EOL>

Loads an entity, optionally checking the cache first

Args:
    name (str): The associated name of the entity
    file_name (str): The location of the entity file
    reload_cache (bool): Whether to refresh all of cache


def _train_and_save(obj, cache, data, print_updates):

obj.train(data)<EOL>if print_updates:<EOL><INDENT>print('<STR_LIT>' + obj.name + '<STR_LIT:.>')<EOL><DEDENT>obj.save(cache)<EOL>

Internal pickleable function used to train objects in another process


def wrapper(*args, **kwargs):<EOL><INDENT>np.random.seed(<NUM_LIT>)<EOL>return func(*args, **kwargs)<EOL><DEDENT>return wrapper<EOL>

Sets the seed of numpy's random number generator


def generate_lowrank_matrix(n=<NUM_LIT:10>, m=<NUM_LIT:20>, k=<NUM_LIT:3>, eta=<NUM_LIT>, seed=<NUM_LIT>):

print("<STR_LIT>")<EOL>global Xtrue, Xobs<EOL>np.random.seed(seed)<EOL>Xtrue = np.sin(np.linspace(<NUM_LIT:0>, <NUM_LIT:2> * np.pi, n)).reshape(-<NUM_LIT:1>, <NUM_LIT:1>).dot(<EOL>np.cos(np.linspace(<NUM_LIT:0>, <NUM_LIT:2> * np.pi, m)).reshape(<NUM_LIT:1>, -<NUM_LIT:1>))<EOL>Xobs = Xtrue + eta * np.random.randn(n, m)<EOL>return Xobs, Xtrue<EOL>

Generate an n-by-m noisy low-rank matrix


class GradientOptimizer(Optimizer):<EOL><INDENT>@wraps(coro)<EOL>def __init__(self, *args, **kwargs):<EOL><INDENT>self.algorithm = coro(*args, **kwargs)<EOL>self.algorithm.send(None)<EOL>self.operators = []<EOL><DEDENT>def set_transform(self, func):<EOL><INDENT>self.transform = compose(destruct, func, self.restruct)<EOL><DEDENT>def minimize(self, f_df, x0, display=sys.stdout, maxiter=<NUM_LIT>):<EOL><INDENT>self.display = display<EOL>self.theta = x0<EOL>xk = self.algorithm.send(destruct(x0).copy())<EOL>store = defaultdict(list)<EOL>runtimes = []<EOL>if len(self.operators) == <NUM_LIT:0>:<EOL><INDENT>self.operators = [proxops.identity()]<EOL><DEDENT>obj, grad = wrap(f_df, x0)<EOL>transform = compose(destruct, *reversed(self.operators), self.restruct)<EOL>self.optional_print(tp.header(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']))<EOL>try:<EOL><INDENT>for k in count():<EOL><INDENT>tstart = perf_counter()<EOL>f = obj(xk)<EOL>df = grad(xk)<EOL>xk = transform(self.algorithm.send(df))<EOL>runtimes.append(perf_counter() - tstart)<EOL>store['<STR_LIT:f>'].append(f)<EOL>self.optional_print(tp.row([k,<EOL>f,<EOL>np.linalg.norm(destruct(df)),<EOL>tp.humantime(runtimes[-<NUM_LIT:1>])]))<EOL>if k >= maxiter:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT>self.optional_print(tp.bottom(<NUM_LIT:4>))<EOL>self.optional_print(u'<STR_LIT>'.format(store['<STR_LIT:f>'][-<NUM_LIT:1>]))<EOL>self.optional_print(u'<STR_LIT>'.format(tp.humantime(sum(runtimes))))<EOL>self.optional_print(u'<STR_LIT>'.format(<EOL>tp.humantime(np.mean(runtimes)),<EOL>tp.humantime(np.std(runtimes)),<EOL>))<EOL>return OptimizeResult({<EOL>'<STR_LIT:x>': self.restruct(xk),<EOL>'<STR_LIT:f>': f,<EOL>'<STR_LIT>': self.restruct(df),<EOL>'<STR_LIT:k>': k,<EOL>'<STR_LIT>': np.array(store['<STR_LIT:f>']),<EOL>})<EOL><DEDENT><DEDENT>return GradientOptimizer<EOL>

Turns a coroutine into a gradient based optimizer.


@gradient_optimizer<EOL>def adam(lr=<NUM_LIT>, beta=(<NUM_LIT>, <NUM_LIT>), epsilon=<NUM_LIT>):

xk = yield<EOL>mk = np.zeros_like(xk)<EOL>vk = np.zeros_like(xk)<EOL>b1, b2 = beta<EOL>for k in count(start=<NUM_LIT:1>):<EOL><INDENT>grad = yield xk<EOL>mk *= b1<EOL>mk += (<NUM_LIT:1.> - b1) * grad<EOL>vk *= b2<EOL>vk += (<NUM_LIT:1.> - b2) * (grad ** <NUM_LIT:2>)<EOL>momentum_norm = mk / (<NUM_LIT:1> - b1 ** k)<EOL>velocity_norm = np.sqrt(vk / (<NUM_LIT:1> - b2 ** k))<EOL>xk -= lr * momentum_norm / (epsilon + velocity_norm)<EOL><DEDENT>

ADAM

Parameters
----------
lr : float, optional
    Learnin rate (Default: 1e-3)

beta : (float, float)
    (Default: (0.9, 0.999))

epsilon : float
    (Default: 1e-8)


@gradient_optimizer<EOL>def rmsprop(lr=<NUM_LIT>, damping=<NUM_LIT>, decay=<NUM_LIT>):

xk = yield<EOL>rms = np.zeros_like(xk)<EOL>for k in count():<EOL><INDENT>grad = yield xk<EOL>rms *= decay<EOL>rms += (<NUM_LIT:1> - decay) * grad**<NUM_LIT:2><EOL>xk -= lr * grad / (damping + np.sqrt(rms))<EOL><DEDENT>

RMSProp

Parameters
----------
lr : float, optional
    Learning rate (Default: 1e-3)

damping : float, optional
    Damping term (Default: 1e-12)

decay : float, optional
    Decay of the learning rate (Default: 0)


xk = yield<EOL>gradients = deque([], nterms)<EOL>for k in count():<EOL><INDENT>grad = yield xk<EOL>gradients.append(grad)<EOL>xk -= lr * np.mean(gradients, axis=<NUM_LIT:0>)<EOL><DEDENT>

Stochastic Average Gradient (SAG)

Parameters
----------
nterms : int, optional
    Number of gradient evaluations to use in the average (Default: 10)

lr : float, optional
    (Default: 1e-3)


@proxify<EOL>def squared_error(x, rho, x_obs):

return (x + x_obs / rho) / (<NUM_LIT:1.> + <NUM_LIT:1.> / rho)<EOL>

Proximal operator for squared error (l2 or Fro. norm)

squared_error(x_obs)

Parameters
----------
x_obs : array_like
    Observed array or matrix that you want to stay close to


return np.maximum(x, <NUM_LIT:0>)<EOL>

Projection onto the non-negative orthant


@proxify<EOL>def lbfgs(x, rho, f_df, maxiter=<NUM_LIT:20>):

def f_df_augmented(theta):<EOL><INDENT>f, df = f_df(theta)<EOL>obj = f + (rho / <NUM_LIT>) * np.linalg.norm(theta - x) ** <NUM_LIT:2><EOL>grad = df + rho * (theta - x)<EOL>return obj, grad<EOL><DEDENT>res = scipy_minimize(f_df_augmented, x, jac=True, method='<STR_LIT>',<EOL>options={'<STR_LIT>': maxiter, '<STR_LIT>': False})<EOL>return res.x<EOL>

Minimize the proximal operator of a given objective using L-BFGS

Parameters
----------
f_df : function
    Returns the objective and gradient of the function to minimize

maxiter : int
    Maximum number of L-BFGS iterations


self.P = A.T.dot(A)<EOL>self.q = A.T.dot(b)<EOL>self.n = self.q.size<EOL>

Proximal operator for solving a linear least squares system, Ax = b

Parameters
----------
A : array_like
    Sensing matrix (Ax = b)

b : array_like
    Responses (Ax = b)


@proxify<EOL>def smooth(x, rho, penalty, axis=<NUM_LIT:0>, newshape=None):

orig_shape = x.shape<EOL>if newshape is not None:<EOL><INDENT>x = x.reshape(newshape)<EOL><DEDENT>n = x.shape[axis]<EOL>lap_op = spdiags([(<NUM_LIT:2> + rho / penalty) * np.ones(n),<EOL>-<NUM_LIT:1> * np.ones(n), -<NUM_LIT:1> * np.ones(n)],<EOL>[<NUM_LIT:0>, -<NUM_LIT:1>, <NUM_LIT:1>], n, n, format='<STR_LIT>')<EOL>A = penalty * lap_op<EOL>b = rho * np.rollaxis(x, axis, <NUM_LIT:0>)<EOL>return np.rollaxis(spsolve(A, b), axis, <NUM_LIT:0>).reshape(orig_shape)<EOL>

Applies a smoothing operator along one dimension

currently only accepts a matrix as input

Parameters
----------
penalty : float

axis : int, optional
    Axis along which to apply the smoothing (Default: 0)

newshape : tuple, optional
    Desired shape of the parameters to apply the nuclear norm to. The given
    parameters are reshaped to an array with this shape, or not reshaped if
    the value of newshape is None. (Default: None)


@proxify<EOL>def columns(x, rho, proxop):

xnext = np.zeros_like(x)<EOL>for ix in range(x.shape[<NUM_LIT:1>]):<EOL><INDENT>xnext[:, ix] = proxop(x[:, ix], rho)<EOL><DEDENT>return xnext<EOL>

Applies a proximal operator to the columns of a matrix


<EOL>u = np.flipud(np.sort(x.ravel()))<EOL>lambdas = (<NUM_LIT:1> - np.cumsum(u)) / (<NUM_LIT:1.> + np.arange(u.size))<EOL>ix = np.where(u + lambdas > <NUM_LIT:0>)[<NUM_LIT:0>].max()<EOL>return np.maximum(x + lambdas[ix], <NUM_LIT:0>)<EOL>

Projection onto the probability simplex

http://arxiv.org/pdf/1309.1541v1.pdf


x, y = theta<EOL>obj = np.sin(x + y) + (x - y)**<NUM_LIT:2> - <NUM_LIT> * x + <NUM_LIT> * y + <NUM_LIT:1><EOL>grad = np.array([np.cos(x + y) + <NUM_LIT:2> * (x - y) - <NUM_LIT>,<EOL>np.cos(x + y) - <NUM_LIT:2> * (x - y) + <NUM_LIT>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = <NUM_LIT:2> * x ** <NUM_LIT:2> - <NUM_LIT> * x ** <NUM_LIT:4> + x ** <NUM_LIT:6> / <NUM_LIT:6> + x * y + y ** <NUM_LIT:2><EOL>grad = np.array([<EOL><NUM_LIT:4> * x - <NUM_LIT> * x ** <NUM_LIT:3> + x ** <NUM_LIT:5> + y,<EOL>x + <NUM_LIT:2> * y<EOL>])<EOL>return obj, grad<EOL>


x, y = theta<EOL>obj = - np.sin(x) * np.sin(x ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20> -np.sin(y) * np.sin(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20><EOL>grad = np.array([<EOL>- np.cos(x) * np.sin(x ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20> - (<NUM_LIT> / np.pi) * x *<EOL>np.sin(x) * np.sin(x ** <NUM_LIT:2> / np.pi) ** <NUM_LIT> * np.cos(x ** <NUM_LIT:2> / np.pi),<EOL>- np.cos(y) * np.sin(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi) ** <NUM_LIT:20> - (<NUM_LIT> / np.pi) * y * np.sin(y) *<EOL>np.sin(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi) ** <NUM_LIT> * np.cos(<NUM_LIT:2> * y ** <NUM_LIT:2> / np.pi),<EOL>])<EOL>return obj, grad<EOL>


def objective(param_scales=(<NUM_LIT:1>, <NUM_LIT:1>), xstar=None, seed=None):

ndim = len(param_scales)<EOL>def decorator(func):<EOL><INDENT>@wraps(func)<EOL>def wrapper(theta):<EOL><INDENT>return func(theta)<EOL><DEDENT>def param_init():<EOL><INDENT>np.random.seed(seed)<EOL>return np.random.randn(ndim,) * np.array(param_scales)<EOL><DEDENT>wrapper.ndim = ndim<EOL>wrapper.param_init = param_init<EOL>wrapper.xstar = xstar<EOL>return wrapper<EOL><DEDENT>return decorator<EOL>

Gives objective functions a number of dimensions and parameter range

    Parameters
    ----------
    param_scales : (int, int)
        Scale (std. dev.) for choosing each parameter

    xstar : array_like
        Optimal parameters


memoized_f_df = lrucache(lambda x: f_df(restruct(x, xref)), size)<EOL>objective = compose(first, memoized_f_df)<EOL>gradient = compose(destruct, second, memoized_f_df)<EOL>return objective, gradient<EOL>

Memoizes an objective + gradient function, and splits it into
two functions that return just the objective and gradient, respectively.

Parameters
----------
f_df : function
    Must be unary (takes a single argument)

xref : list, dict, or array_like
    The form of the parameters

size : int, optional
    Size of the cache (Default=1)


def check_grad(f_df, xref, stepsize=<NUM_LIT>, tol=<NUM_LIT>, width=<NUM_LIT:15>, style='<STR_LIT>', out=sys.stdout):

CORRECT = u'<STR_LIT>'<EOL>INCORRECT = u'<STR_LIT>'<EOL>obj, grad = wrap(f_df, xref, size=<NUM_LIT:0>)<EOL>x0 = destruct(xref)<EOL>df = grad(x0)<EOL>out.write(tp.header(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"], width=width, style=style) + "<STR_LIT:\n>")<EOL>out.flush()<EOL>def parse_error(number):<EOL><INDENT>failure = "<STR_LIT>"<EOL>passing = "<STR_LIT>"<EOL>warning = "<STR_LIT>"<EOL>end = "<STR_LIT>"<EOL>base = "<STR_LIT>"<EOL>if error < <NUM_LIT:0.1> * tol:<EOL><INDENT>return base.format(passing, error, end)<EOL><DEDENT>elif error < tol:<EOL><INDENT>return base.format(warning, error, end)<EOL><DEDENT>else:<EOL><INDENT>return base.format(failure, error, end)<EOL><DEDENT><DEDENT>num_errors = <NUM_LIT:0><EOL>for j in range(x0.size):<EOL><INDENT>dx = np.zeros(x0.size)<EOL>dx[j] = stepsize<EOL>df_approx = (obj(x0 + dx) - obj(x0 - dx)) / (<NUM_LIT:2> * stepsize)<EOL>df_analytic = df[j]<EOL>abs_error = np.linalg.norm(df_approx - df_analytic)<EOL>error = abs_error if np.allclose(abs_error, <NUM_LIT:0>) else abs_error /(np.linalg.norm(df_analytic) + np.linalg.norm(df_approx))<EOL>num_errors += error >= tol<EOL>errstr = CORRECT if error < tol else INCORRECT<EOL>out.write(tp.row([df_approx, df_analytic, parse_error(error) + '<STR_LIT:U+0020>' + errstr],<EOL>width=width, style=style) + "<STR_LIT:\n>")<EOL>out.flush()<EOL><DEDENT>out.write(tp.bottom(<NUM_LIT:3>, width=width, style=style) + "<STR_LIT:\n>")<EOL>return num_errors<EOL>

Compares the numerical gradient to the analytic gradient

Parameters
----------
f_df : function
    The analytic objective and gradient function to check

x0 : array_like
    Parameter values to check the gradient at

stepsize : float, optional
    Stepsize for the numerical gradient. Too big and this will poorly estimate the gradient.
    Too small and you will run into precision issues (default: 1e-6)

tol : float, optional
    Tolerance to use when coloring correct/incorrect gradients (default: 1e-5)

width : int, optional
    Width of the table columns (default: 15)

style : string, optional
    Style of the printed table, see tableprint for a list of styles (default: 'round')


try:<EOL><INDENT>return psycopg2.connect(**db_state['<STR_LIT>'])<EOL><DEDENT>except psycopg2.OperationalError as exc:<EOL><INDENT>nosuch_db = '<STR_LIT>' % db_state['<STR_LIT>']['<STR_LIT>']<EOL>if nosuch_db in str(exc):<EOL><INDENT>try:<EOL><INDENT>master = psycopg2.connect(database='<STR_LIT>')<EOL>master.rollback()<EOL>master.autocommit = True<EOL>cursor = master.cursor()<EOL>cursor.execute('<STR_LIT>' % db_state['<STR_LIT>']['<STR_LIT>'])<EOL>cursor.close()<EOL>master.close()<EOL><DEDENT>except psycopg2.Error as exc:<EOL><INDENT>message = ('<STR_LIT>'<EOL>+ db_state['<STR_LIT>']['<STR_LIT>']<EOL>+ '<STR_LIT>' % exc)<EOL>raise RuntimeError(message)<EOL><DEDENT>else:<EOL><INDENT>conn = psycopg2.connect(**db_state['<STR_LIT>'])<EOL>db_state['<STR_LIT>'] = True<EOL>return conn<EOL><DEDENT><DEDENT><DEDENT>


def __init__(self, connection, table, xform):

super(RenameReplace, self).__init__(connection, table)<EOL>self.xform = xform<EOL>

xform must be a function which translates old
names to new ones, used on tables & pk constraints


return p.startswith('<STR_LIT:[>')<EOL>

Is an argument of the URL.

>>> _is_url_arg('[idAction]')
True
>>> _is_url_arg('actions')
False


return line.split('<STR_LIT:U+0020>', <NUM_LIT:1>)[<NUM_LIT:0>] in HTTP_METHODS<EOL>

Is a definition of a Trello endpoint.

>>> _is_api_definition('GET /1/actions/[idAction]')
True
>>> _is_api_definition('action')
False


ep = requests.get(TRELLO_API_DOC).content<EOL>root = html.fromstring(ep)<EOL>links = root.xpath('<STR_LIT>')<EOL>pages = [requests.get(TRELLO_API_DOC + u)<EOL>for u in links if u.endswith('<STR_LIT>')]<EOL>endpoints = []<EOL>for page in pages:<EOL><INDENT>root = html.fromstring(page.content)<EOL>sections = root.xpath('<STR_LIT>')<EOL>for sec in sections:<EOL><INDENT>ep_html = etree.tostring(sec).decode('<STR_LIT:utf-8>')<EOL>ep_text = html2text(ep_html).splitlines()<EOL>match = EP_DESC_REGEX.match(ep_text[<NUM_LIT:0>])<EOL>if not match:<EOL><INDENT>continue<EOL><DEDENT>ep_method, ep_url = match.groups()<EOL>ep_text[<NUM_LIT:0>] = '<STR_LIT:U+0020>'.join([ep_method, ep_url])<EOL>ep_doc = b64encode(gzip.compress('<STR_LIT:\n>'.join(ep_text).encode('<STR_LIT:utf-8>')))<EOL>endpoints.append((ep_method, ep_url, ep_doc))<EOL><DEDENT><DEDENT>print(yaml.dump(create_tree(endpoints)))<EOL>


def upper2underscore(text):<EOL><INDENT>for char in text:<EOL><INDENT>if char.islower():<EOL><INDENT>yield char<EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT:_>'<EOL>if char.isalpha():<EOL><INDENT>yield char.lower()<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return '<STR_LIT>'.join(upper2underscore(url))<EOL>

Translate camelCase into underscore format.

>>> _camelcase_to_underscore('minutesBetweenSummaries')
'minutes_between_summaries'


if self._api_arg:<EOL><INDENT>mypart = str(self._api_arg)<EOL><DEDENT>else:<EOL><INDENT>mypart = self._name<EOL><DEDENT>if self._parent:<EOL><INDENT>return '<STR_LIT:/>'.join(filter(None, [self._parent._url, mypart]))<EOL><DEDENT>else:<EOL><INDENT>return mypart<EOL><DEDENT>

Resolve the URL to this point.

>>> trello = TrelloAPIV1('APIKEY')
>>> trello.batch._url
'1/batch'
>>> trello.boards(board_id='BOARD_ID')._url
'1/boards/BOARD_ID'
>>> trello.boards(board_id='BOARD_ID')(field='FIELD')._url
'1/boards/BOARD_ID/FIELD'
>>> trello.boards(board_id='BOARD_ID').cards(filter='FILTER')._url
'1/boards/BOARD_ID/cards/FILTER'


def get_partial_api(key, token=None):<EOL><INDENT>return TrelloAPI(ENDPOINTS[version], version, key, token=token)<EOL><DEDENT>get_partial_api.__doc__ ="""<STR_LIT>""".format(version)<EOL>return get_partial_api<EOL>

Generates a factory function to instantiate the API with the given
version.


if isinstance(other, Range):<EOL><INDENT>start = self.start.replace(tzinfo=other.start.tz) if other.start.tz and self.start.tz is None else self.start<EOL>end = self.end.replace(tzinfo=other.end.tz) if other.end.tz and self.end.tz is None else self.end<EOL>if start == other.start and end == other.end:<EOL><INDENT>return <NUM_LIT:0> <EOL><DEDENT>elif start < other.start:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>elif isinstance(other, Date):<EOL><INDENT>if other.tz and self.start.tz is None:<EOL><INDENT>return <NUM_LIT:0> if other == self.start.replace(tzinfo=other.tz) else -<NUM_LIT:1> if other > self.start.replace(tzinfo=other.start.tz) else <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0> if other == self.start else -<NUM_LIT:1> if other > self.start else <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return self.cmp(Range(other, tz=self.start.tz))<EOL><DEDENT>


s, e = copy(self.start), copy(self.end)<EOL>if from_start:<EOL><INDENT>e = s + by<EOL><DEDENT>else:<EOL><INDENT>s = e - by<EOL><DEDENT>return Range(s, e)<EOL>

Cuts this object from_start to the number requestd
        returns new instance


return str(self.date)<EOL>

Returns date in representation of `%x %X` ie `2013-02-17 00:00:00`


if self.date != '<STR_LIT>':<EOL><INDENT>return Date(self.date.replace(**k))<EOL><DEDENT>else:<EOL><INDENT>return Date('<STR_LIT>')<EOL><DEDENT>


if map is None:<EOL><INDENT>map = asyncore.socket_map<EOL><DEDENT>pollster = select.epoll()<EOL>if map:<EOL><INDENT>for fd, obj in iteritems(map):<EOL><INDENT>flags = <NUM_LIT:0><EOL>if obj.readable():<EOL><INDENT>flags |= select.POLLIN | select.POLLPRI<EOL><DEDENT>if obj.writable():<EOL><INDENT>flags |= select.POLLOUT<EOL><DEDENT>if flags:<EOL><INDENT>flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL<EOL>pollster.register(fd, flags)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>r = pollster.poll(timeout)<EOL><DEDENT>except select.error as err:<EOL><INDENT>if err.args[<NUM_LIT:0>] != EINTR:<EOL><INDENT>raise<EOL><DEDENT>r = []<EOL><DEDENT>for fd, flags in r:<EOL><INDENT>obj = map.get(fd)<EOL>if obj is None:<EOL><INDENT>continue<EOL><DEDENT>asyncore.readwrite(obj, flags)<EOL><DEDENT><DEDENT>

A poller which uses epoll(), supported on Linux 2.5.44 and newer

Borrowed from here:
https://github.com/m13253/python-asyncore-epoll/blob/master/asyncore_epoll.py#L200


Start serving HTTP requests

This method blocks the current thread.

:param poll_interval: polling timeout
:return:


self.iterator = iter(self.result)<EOL>

Get WSGI response iterator for sending in handle_write


handler_class.ws_handler_class = ws_handler_class<EOL>handler_class.ws_path = ws_path<EOL>httpd = server_class((host, port), RequestHandlerClass=handler_class)<EOL>httpd.set_app(app)<EOL>return httpd<EOL>

Create server instance with an optional WebSocket handler

    For pure WebSocket server ``app`` may be ``None`` but an attempt to access
    any path other than ``ws_path`` will cause server error.

    :param host: hostname or IP
    :type host: str
    :param port: server port
    :type port: int
    :param app: WSGI application
    :param server_class: WSGI server class, defaults to AsyncWsgiServer
    :param handler_class: WSGI handler class, defaults to AsyncWsgiHandler
    :param ws_handler_class: WebSocket hanlder class, defaults to ``None``
    :param ws_path: WebSocket path on the server, defaults to '/ws'
    :type ws_path: str, optional
    :return: initialized server instance


pass<EOL>

Called when a websocket server gets a Close frame from a client.


opcode = BINARY<EOL>if _check_unicode(data):<EOL><INDENT>opcode = TEXT<EOL><DEDENT>self._sendMessage(True, opcode, data)<EOL>

Send the start of a data fragment stream to a websocket client.
Subsequent data should be sent using sendFragment().
A fragment stream is completed when sendFragmentEnd() is called.

If data is a unicode object then the frame is sent as Text.
If the data is a bytearray object then the frame is sent as Binary.


opcode = BINARY<EOL>if _check_unicode(data):<EOL><INDENT>opcode = TEXT<EOL><DEDENT>self._sendMessage(False, opcode, data)<EOL>

Send websocket data frame to the client.

If data is a unicode object then the frame is sent as Text.
If the data is a bytearray object then the frame is sent as Binary.


def close(self, status=<NUM_LIT:1000>, reason=u'<STR_LIT>'):

try:<EOL><INDENT>if self.closed is False:<EOL><INDENT>close_msg = bytearray()<EOL>close_msg.extend(struct.pack("<STR_LIT>", status))<EOL>if _check_unicode(reason):<EOL><INDENT>close_msg.extend(reason.encode('<STR_LIT:utf-8>'))<EOL><DEDENT>else:<EOL><INDENT>close_msg.extend(reason)<EOL><DEDENT>self._sendMessage(False, CLOSE, close_msg)<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>self.closed = True<EOL><DEDENT>

Send Close frame to the client. The underlying socket is only closed
when the client acknowledges the Close frame.

status is the closing identifier.
reason is the reason for the close.


@classmethod<EOL><INDENT>def get(cls, update_dict=None, remove_fields=None):<DEDENT>

cls.demo_course_count += <NUM_LIT:1><EOL>course_copy = copy.deepcopy(cls.DEMO_COURSE)<EOL>if update_dict:<EOL><INDENT>if "<STR_LIT:content>" in update_dict:<EOL><INDENT>course_copy["<STR_LIT:content>"].update(update_dict["<STR_LIT:content>"])<EOL>del update_dict["<STR_LIT:content>"]<EOL><DEDENT>course_copy.update(update_dict)<EOL><DEDENT>course_copy.update({"<STR_LIT:id>": "<STR_LIT>".format(course_copy["<STR_LIT:id>"], cls.demo_course_count)})<EOL>if remove_fields:<EOL><INDENT>for remove_field in remove_fields:<EOL><INDENT>if remove_field in course_copy:<EOL><INDENT>del course_copy[remove_field]<EOL><DEDENT><DEDENT><DEDENT>return course_copy<EOL>


return isinstance(self.searcher, ElasticSearchEngine)<EOL>

check search engine implementation, to manage cleanup differently


def assert_results_returned_event(self, search_term, size, page, total):

returned_results_call = self.mock_tracker.emit.mock_calls[<NUM_LIT:1>]  <EOL>expected_result = call('<STR_LIT>', {<EOL>"<STR_LIT>": six.text_type(search_term),<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": total,<EOL>})<EOL>self.assertEqual(expected_result, returned_results_call)<EOL>

Ensures an results returned event was emitted


self.assertFalse(self.mock_tracker.emit.called)<EOL>

Ensures no events were emitted since the last event related assertion


return "<STR_LIT>"<EOL>

additional property that should appear within processed results


return "<STR_LIT>" in self._results_fields<EOL>


raise exceptions.ElasticsearchException("<STR_LIT>")<EOL>


address = '<STR_LIT>'<EOL>response = Client().post(address, body)<EOL>return getattr(response, "<STR_LIT>", <NUM_LIT>), json.loads(getattr(response, "<STR_LIT:content>", None))<EOL>

Helper method to post the request and process the response


address = '<STR_LIT:/>' if course_id is None else '<STR_LIT>'.format(course_id)<EOL>response = Client().post(address, body)<EOL>return getattr(response, "<STR_LIT>", <NUM_LIT>), json.loads(getattr(response, "<STR_LIT:content>", None))<EOL>


if self._searcher is None:<EOL><INDENT>self._searcher = SearchEngine.get_search_engine(TEST_INDEX_NAME)<EOL><DEDENT>return self._searcher<EOL>


if "<STR_LIT:T>" in json_date_string_value:<EOL><INDENT>if "<STR_LIT:.>" in json_date_string_value:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>if json_date_string_value.endswith("<STR_LIT>"):<EOL><INDENT>format_string += "<STR_LIT>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>return datetime.strptime(<EOL>json_date_string_value,<EOL>format_string<EOL>)<EOL>

converts json date string to date object


if MockSearchEngine._disabled:<EOL><INDENT>return {<EOL>"<STR_LIT>": <NUM_LIT:10>,<EOL>"<STR_LIT>": <NUM_LIT:0>,<EOL>"<STR_LIT>": <NUM_LIT:0>,<EOL>"<STR_LIT>": []<EOL>}<EOL><DEDENT>documents_to_search = []<EOL>if "<STR_LIT>" in kwargs:<EOL><INDENT>documents_to_search = MockSearchEngine.load_doc_type(self.index_name, kwargs["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>index = MockSearchEngine.load_index(self.index_name)<EOL>for doc_type in index:<EOL><INDENT>documents_to_search.extend(index[doc_type])<EOL><DEDENT><DEDENT>if field_dictionary:<EOL><INDENT>documents_to_search = _filter_intersection(documents_to_search, field_dictionary)<EOL><DEDENT>if filter_dictionary:<EOL><INDENT>documents_to_search = _filter_intersection(documents_to_search, filter_dictionary, True)<EOL><DEDENT>if query_string:<EOL><INDENT>documents_to_search = _process_query_string(documents_to_search, query_string)<EOL><DEDENT>if "<STR_LIT>" in kwargs:<EOL><INDENT>if not exclude_dictionary:<EOL><INDENT>exclude_dictionary = {}<EOL><DEDENT>if "<STR_LIT:id>" not in exclude_dictionary:<EOL><INDENT>exclude_dictionary["<STR_LIT:id>"] = []<EOL><DEDENT>exclude_dictionary["<STR_LIT:id>"].extend(kwargs["<STR_LIT>"])<EOL><DEDENT>if exclude_dictionary:<EOL><INDENT>documents_to_search = _process_exclude_dictionary(documents_to_search, exclude_dictionary)<EOL><DEDENT>def score_documents(documents_to_search):<EOL><INDENT>"""<STR_LIT>"""<EOL>search_results = []<EOL>max_score = <NUM_LIT:0><EOL>while documents_to_search:<EOL><INDENT>current_doc = documents_to_search[<NUM_LIT:0>]<EOL>score = len([d for d in documents_to_search if d == current_doc])<EOL>if score > max_score:<EOL><INDENT>max_score = score<EOL><DEDENT>documents_to_search = [d for d in documents_to_search if d != current_doc]<EOL>data = copy.copy(current_doc)<EOL>search_results.append(<EOL>{<EOL>"<STR_LIT>": score,<EOL>"<STR_LIT:data>": data,<EOL>}<EOL>)<EOL><DEDENT>return search_results, max_score<EOL><DEDENT>search_results, max_score = score_documents(documents_to_search)<EOL>results = MockSearchEngine._paginate_results(<EOL>kwargs["<STR_LIT:size>"] if "<STR_LIT:size>" in kwargs else None,<EOL>kwargs["<STR_LIT>"] if "<STR_LIT>" in kwargs else None,<EOL>sorted(search_results, key=lambda k: k["<STR_LIT>"])<EOL>)<EOL>response = {<EOL>"<STR_LIT>": <NUM_LIT:10>,<EOL>"<STR_LIT>": len(search_results),<EOL>"<STR_LIT>": max_score,<EOL>"<STR_LIT>": results<EOL>}<EOL>if facet_terms:<EOL><INDENT>response["<STR_LIT>"] = _count_facet_values(documents_to_search, facet_terms)<EOL><DEDENT>return response<EOL>

Perform search upon documents within index


if not MockSearchEngine._disabled:<EOL><INDENT>doc_ids = [s["<STR_LIT:id>"] for s in sources if "<STR_LIT:id>" in s]<EOL>MockSearchEngine.remove_documents(self.index_name, doc_type, doc_ids)<EOL>MockSearchEngine.add_documents(self.index_name, doc_type, sources)<EOL><DEDENT>

Add/update documents of given type to the index


cls._load_from_file()<EOL>if index_name not in cls._mock_elastic:<EOL><INDENT>cls._mock_elastic[index_name] = {}<EOL>cls._write_to_file()<EOL><DEDENT>return cls._mock_elastic[index_name]<EOL>

load the index, if necessary from the backed file


if not MockSearchEngine._disabled:<EOL><INDENT>MockSearchEngine.remove_documents(self.index_name, doc_type, doc_ids)<EOL><DEDENT>

Remove documents of type with given ids from the index


@staticmethod<EOL><INDENT>def _paginate_results(size, from_, raw_results):<DEDENT>

results = raw_results<EOL>if size:<EOL><INDENT>start = <NUM_LIT:0><EOL>if from_ is not None:<EOL><INDENT>start = from_<EOL><DEDENT>results = raw_results[start:start + size]<EOL><DEDENT>return results<EOL>


for exclude_property in exclude_dictionary:<EOL><INDENT>exclude_values = exclude_dictionary[exclude_property]<EOL>if not isinstance(exclude_values, list):<EOL><INDENT>exclude_values = [exclude_values]<EOL><DEDENT>documents_to_search = [<EOL>document<EOL>for document in documents_to_search<EOL>if document.get(exclude_property) not in exclude_values<EOL>]<EOL><DEDENT>return documents_to_search<EOL>

remove results that have fields that match in the exclude_dictionary


file_name = cls._backing_file(create_if_missing)<EOL>if file_name:<EOL><INDENT>with open(file_name, "<STR_LIT>") as dict_file:<EOL><INDENT>json.dump(cls._mock_elastic, dict_file, cls=DjangoJSONEncoder)<EOL><DEDENT><DEDENT>

write the index dict to the backing file


cls._mock_elastic = {}<EOL>cls._write_to_file()<EOL>

Clean out the dictionary for test resets


initializer = _load_class(getattr(settings, "<STR_LIT>", None), cls)()<EOL>return initializer.initialize(**kwargs)<EOL>

Called from within search handler
Finds desired subclass and calls initialize method


pass<EOL>


return {<EOL>field_key: request.POST[field_key]<EOL>for field_key in request.POST<EOL>if field_key in course_discovery_filter_fields()<EOL>}<EOL>

Create separate dictionary of supported filter values provided


<EOL>SearchInitializer.set_search_enviroment(request=request, course_id=course_id)<EOL>results = {<EOL>"<STR_LIT:error>": _("<STR_LIT>")<EOL>}<EOL>status_code = <NUM_LIT><EOL>search_term = request.POST.get("<STR_LIT>", None)<EOL>try:<EOL><INDENT>if not search_term:<EOL><INDENT>raise ValueError(_('<STR_LIT>'))<EOL><DEDENT>size, from_, page = _process_pagination_values(request)<EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>}<EOL>)<EOL>results = perform_search(<EOL>search_term,<EOL>user=request.user,<EOL>size=size,<EOL>from_=from_,<EOL>course_id=course_id<EOL>)<EOL>status_code = <NUM_LIT:200><EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": results["<STR_LIT>"],<EOL>}<EOL>)<EOL><DEDENT>except ValueError as invalid_err:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": six.text_type(invalid_err)<EOL>}<EOL>log.debug(six.text_type(invalid_err))<EOL><DEDENT>except QueryParseError:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>')<EOL>}<EOL><DEDENT>except Exception as err:  <EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>').format(search_string=search_term)<EOL>}<EOL>log.exception(<EOL>'<STR_LIT>',<EOL>search_term,<EOL>request.user.id,<EOL>err<EOL>)<EOL><DEDENT>return JsonResponse(results, status=status_code)<EOL>

Search view for http requests

Args:
    request (required) - django request object
    course_id (optional) - course_id within which to restrict search

Returns:
    http json response with the following fields
        "took" - how many seconds the operation took
        "total" - how many results were found
        "max_score" - maximum score from these results
        "results" - json array of result documents

        or

        "error" - displayable information about an error that occured on the server

POST Params:
    "search_string" (required) - text upon which to search
    "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
    "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)


results = {<EOL>"<STR_LIT:error>": _("<STR_LIT>")<EOL>}<EOL>status_code = <NUM_LIT><EOL>search_term = request.POST.get("<STR_LIT>", None)<EOL>try:<EOL><INDENT>size, from_, page = _process_pagination_values(request)<EOL>field_dictionary = _process_field_values(request)<EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>}<EOL>)<EOL>results = course_discovery_search(<EOL>search_term=search_term,<EOL>size=size,<EOL>from_=from_,<EOL>field_dictionary=field_dictionary,<EOL>)<EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": results["<STR_LIT>"],<EOL>}<EOL>)<EOL>status_code = <NUM_LIT:200><EOL><DEDENT>except ValueError as invalid_err:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": six.text_type(invalid_err)<EOL>}<EOL>log.debug(six.text_type(invalid_err))<EOL><DEDENT>except QueryParseError:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>')<EOL>}<EOL><DEDENT>except Exception as err:  <EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>').format(search_string=search_term)<EOL>}<EOL>log.exception(<EOL>'<STR_LIT>',<EOL>search_term,<EOL>request.user.id,<EOL>err<EOL>)<EOL><DEDENT>return JsonResponse(results, status=status_code)<EOL>

Search for courses

Args:
    request (required) - django request object

Returns:
    http json response with the following fields
        "took" - how many seconds the operation took
        "total" - how many results were found
        "max_score" - maximum score from these resutls
        "results" - json array of result documents

        or

        "error" - displayable information about an error that occured on the server

POST Params:
    "search_string" (optional) - text with which to search for courses
    "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
    "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)


field_dictionary = {}<EOL>if "<STR_LIT>" in kwargs and kwargs["<STR_LIT>"]:<EOL><INDENT>field_dictionary["<STR_LIT>"] = kwargs["<STR_LIT>"]<EOL><DEDENT>return field_dictionary<EOL>

base implementation which add course if provided


return {"<STR_LIT>": DateRange(None, datetime.utcnow())}<EOL>

base implementation which filters via start_date


def _check_mappings(self, doc_type, body):

<EOL>exclude_fields = ["<STR_LIT:content>"]<EOL>field_properties = getattr(settings, "<STR_LIT>", {})<EOL>def field_property(field_name, field_value):<EOL><INDENT>"""<STR_LIT>"""<EOL>prop_val = None<EOL>if field_name in field_properties:<EOL><INDENT>prop_val = field_properties[field_name]<EOL><DEDENT>elif isinstance(field_value, dict):<EOL><INDENT>props = {fn: field_property(fn, field_value[fn]) for fn in field_value}<EOL>prop_val = {"<STR_LIT>": props}<EOL><DEDENT>else:<EOL><INDENT>prop_val = {<EOL>"<STR_LIT:type>": "<STR_LIT:string>",<EOL>"<STR_LIT:index>": "<STR_LIT>",<EOL>}<EOL><DEDENT>return prop_val<EOL><DEDENT>new_properties = {<EOL>field: field_property(field, value)<EOL>for field, value in body.items()<EOL>if (field not in exclude_fields) and (field not in self._get_mappings(doc_type).get('<STR_LIT>', {}))<EOL>}<EOL>if new_properties:<EOL><INDENT>self._es.indices.put_mapping(<EOL>index=self.index_name,<EOL>doc_type=doc_type,<EOL>body={<EOL>doc_type: {<EOL>"<STR_LIT>": new_properties,<EOL>}<EOL>}<EOL>)<EOL>self._clear_mapping(doc_type)<EOL><DEDENT>

We desire to index content so that anything we want to be textually searchable(and therefore needing to be
analysed), but the other fields are designed to be filters, and only require an exact match. So, we want to
set up the mappings for these fields as "not_analyzed" - this will allow our filters to work faster because
they only have to work off exact matches


ElasticSearchEngine.set_mappings(self.index_name, doc_type, {})<EOL>

Remove the cached mappings, so that they get loaded from ES next time they are requested


elastic_facets = {}<EOL>for facet in facet_terms:<EOL><INDENT>facet_term = {"<STR_LIT>": facet}<EOL>if facet_terms[facet]:<EOL><INDENT>for facet_option in facet_terms[facet]:<EOL><INDENT>facet_term[facet_option] = facet_terms[facet][facet_option]<EOL><DEDENT><DEDENT>elastic_facets[facet] = {<EOL>"<STR_LIT>": facet_term<EOL>}<EOL><DEDENT>return elastic_facets<EOL>

We have a list of terms with which we return facets


def translate_result(result):<EOL><INDENT>"""<STR_LIT>"""<EOL>translated_result = copy.copy(result)<EOL>data = translated_result.pop("<STR_LIT>")<EOL>translated_result.update({<EOL>"<STR_LIT:data>": data,<EOL>"<STR_LIT>": translated_result["<STR_LIT>"]<EOL>})<EOL>return translated_result<EOL><DEDENT>def translate_facet(result):<EOL><INDENT>"""<STR_LIT>"""<EOL>terms = {term["<STR_LIT>"]: term["<STR_LIT:count>"] for term in result["<STR_LIT>"]}<EOL>return {<EOL>"<STR_LIT>": terms,<EOL>"<STR_LIT>": result["<STR_LIT>"],<EOL>"<STR_LIT>": result["<STR_LIT>"],<EOL>}<EOL><DEDENT>results = [translate_result(hit) for hit in es_response["<STR_LIT>"]["<STR_LIT>"]]<EOL>response = {<EOL>"<STR_LIT>": es_response["<STR_LIT>"],<EOL>"<STR_LIT>": es_response["<STR_LIT>"]["<STR_LIT>"],<EOL>"<STR_LIT>": es_response["<STR_LIT>"]["<STR_LIT>"],<EOL>"<STR_LIT>": results,<EOL>}<EOL>if "<STR_LIT>" in es_response:<EOL><INDENT>response["<STR_LIT>"] = {facet: translate_facet(es_response["<STR_LIT>"][facet]) for facet in es_response["<STR_LIT>"]}<EOL><DEDENT>return response<EOL>

Provide resultset in our desired format from elasticsearch results


def remove(self, doc_type, doc_ids, **kwargs):

try:<EOL><INDENT>actions = []<EOL>for doc_id in doc_ids:<EOL><INDENT>log.debug("<STR_LIT>", doc_type, doc_id)<EOL>action = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>"<STR_LIT>": self.index_name,<EOL>"<STR_LIT>": doc_type,<EOL>"<STR_LIT>": doc_id<EOL>}<EOL>actions.append(action)<EOL><DEDENT>bulk(self._es, actions, **kwargs)<EOL><DEDENT>except BulkIndexError as ex:<EOL><INDENT>valid_errors = [error for error in ex.errors if error['<STR_LIT>']['<STR_LIT:status>'] != <NUM_LIT>]<EOL>if valid_errors:<EOL><INDENT>log.exception("<STR_LIT>")<EOL>raise<EOL><DEDENT><DEDENT>

Implements call to remove the documents from the index


indexing_errors_log = []<EOL>for indexing_error in indexing_errors:<EOL><INDENT>indexing_errors_log.append(str(indexing_error))<EOL><DEDENT>raise exceptions.ElasticsearchException('<STR_LIT:U+002CU+0020>'.join(indexing_errors_log))<EOL>

Logs indexing errors and raises a general ElasticSearch Exception


return "<STR_LIT>".format(<EOL>index_name,<EOL>doc_type<EOL>)<EOL>


return [_get_filter_field(field, field_value) for field, field_value in field_dictionary.items()]<EOL>

We have a field_dictionary - we match the values using a "term" filter in elasticsearch


<EOL>not_properties = []<EOL>for exclude_property in exclude_dictionary:<EOL><INDENT>exclude_values = exclude_dictionary[exclude_property]<EOL>if not isinstance(exclude_values, list):<EOL><INDENT>exclude_values = [exclude_values]<EOL><DEDENT>not_properties.extend([{"<STR_LIT>": {exclude_property: exclude_value}} for exclude_value in exclude_values])<EOL><DEDENT>if not not_properties:<EOL><INDENT>return {}<EOL><DEDENT>return {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": not_properties<EOL>}<EOL>}<EOL>}<EOL>

Based on values in the exclude_dictionary generate a list of term queries that
will filter out unwanted results.


def field_item(field):<EOL><INDENT>"""<STR_LIT>"""<EOL>return {<EOL>"<STR_LIT>": {<EOL>field: field_dictionary[field]<EOL>}<EOL>}<EOL><DEDENT>return [field_item(field) for field in field_dictionary]<EOL>

We have a field_dictionary - we want to match the values for an elasticsearch "match" query
This is only potentially useful when trying to tune certain search operations


filter_field = None<EOL>if isinstance(field_value, ValueRange):<EOL><INDENT>range_values = {}<EOL>if field_value.lower:<EOL><INDENT>range_values.update({"<STR_LIT>": field_value.lower_string})<EOL><DEDENT>if field_value.upper:<EOL><INDENT>range_values.update({"<STR_LIT>": field_value.upper_string})<EOL><DEDENT>filter_field = {<EOL>"<STR_LIT>": {<EOL>field_name: range_values<EOL>}<EOL>}<EOL><DEDENT>elif _is_iterable(field_value):<EOL><INDENT>filter_field = {<EOL>"<STR_LIT>": {<EOL>field_name: field_value<EOL>}<EOL>}<EOL><DEDENT>else:<EOL><INDENT>filter_field = {<EOL>"<STR_LIT>": {<EOL>field_name: field_value<EOL>}<EOL>}<EOL><DEDENT>return filter_field<EOL>

Return field to apply into filter, if an array then use a range, otherwise look for a term match


def index(self, doc_type, sources, **kwargs):

try:<EOL><INDENT>actions = []<EOL>for source in sources:<EOL><INDENT>self._check_mappings(doc_type, source)<EOL>id_ = source['<STR_LIT:id>'] if '<STR_LIT:id>' in source else None<EOL>log.debug("<STR_LIT>", doc_type, id_)<EOL>action = {<EOL>"<STR_LIT>": self.index_name,<EOL>"<STR_LIT>": doc_type,<EOL>"<STR_LIT>": id_,<EOL>"<STR_LIT>": source<EOL>}<EOL>actions.append(action)<EOL><DEDENT>_, indexing_errors = bulk(<EOL>self._es,<EOL>actions,<EOL>**kwargs<EOL>)<EOL>if indexing_errors:<EOL><INDENT>ElasticSearchEngine.log_indexing_error(indexing_errors)<EOL><DEDENT><DEDENT>except Exception as ex:<EOL><INDENT>log.exception("<STR_LIT>", str(ex))<EOL>raise<EOL><DEDENT>

Implements call to add documents to the ES index
Note the call to _check_mappings which will setup fields with the desired mappings


if class_path is None:<EOL><INDENT>return default<EOL><DEDENT>component = class_path.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>result_processor = getattr(<EOL>importlib.import_module(component[<NUM_LIT:0>]),<EOL>component[<NUM_LIT:1>],<EOL>default<EOL>) if len(component) > <NUM_LIT:1> else default<EOL>return result_processor<EOL>

Loads the class from the class_path string


return self._lower.isoformat()<EOL>

use isoformat for _lower date's string format


<EOL>(field_dictionary, filter_dictionary, exclude_dictionary) = SearchFilterGenerator.generate_field_filters(<EOL>user=user,<EOL>course_id=course_id<EOL>)<EOL>searcher = SearchEngine.get_search_engine(getattr(settings, "<STR_LIT>", "<STR_LIT>"))<EOL>if not searcher:<EOL><INDENT>raise NoSearchEngineError("<STR_LIT>")<EOL><DEDENT>results = searcher.search_string(<EOL>search_term,<EOL>field_dictionary=field_dictionary,<EOL>filter_dictionary=filter_dictionary,<EOL>exclude_dictionary=exclude_dictionary,<EOL>size=size,<EOL>from_=from_,<EOL>doc_type="<STR_LIT>",<EOL>)<EOL>for result in results["<STR_LIT>"]:<EOL><INDENT>result["<STR_LIT:data>"] = SearchResultProcessor.process_result(result["<STR_LIT:data>"], search_term, user)<EOL><DEDENT>results["<STR_LIT>"] = len([r for r in results["<STR_LIT>"] if r["<STR_LIT:data>"] is None])<EOL>results["<STR_LIT>"] = [r for r in results["<STR_LIT>"] if r["<STR_LIT:data>"] is not None]<EOL>return results<EOL>

Call the search engine with the appropriate parameters


raise NotImplementedError<EOL>

This operation is called to search for matching documents within the search index


raise NotImplementedError<EOL>

This operation is called to remove documents of given type from the search index


def search_fields(self, field_dictionary, **kwargs):

return self.search(field_dictionary=field_dictionary, **kwargs)<EOL>

Helper function when primary search is for a set of matching fields


search_engine_class = _load_class(getattr(settings, "<STR_LIT>", None), None)<EOL>return search_engine_class(index=index) if search_engine_class else None<EOL>

Returns the desired implementor (defined in settings)


raise NotImplementedError<EOL>

This operation is called to add documents of given type to the search index


if "<STR_LIT:content>" not in self._results_fields:<EOL><INDENT>return None<EOL><DEDENT>match_phrases = [self._match_phrase]<EOL>if six.PY2:<EOL><INDENT>separate_phrases = [<EOL>phrase.decode('<STR_LIT:utf-8>')<EOL>for phrase in shlex.split(self._match_phrase.encode('<STR_LIT:utf-8>'))<EOL>]<EOL><DEDENT>else:<EOL><INDENT>separate_phrases = [<EOL>phrase<EOL>for phrase in shlex.split(self._match_phrase)<EOL>]<EOL><DEDENT>if len(separate_phrases) > <NUM_LIT:1>:<EOL><INDENT>match_phrases.extend(separate_phrases)<EOL><DEDENT>else:<EOL><INDENT>match_phrases = separate_phrases<EOL><DEDENT>matches = SearchResultProcessor.find_matches(<EOL>SearchResultProcessor.strings_in_dictionary(self._results_fields["<STR_LIT:content>"]),<EOL>match_phrases,<EOL>DESIRED_EXCERPT_LENGTH<EOL>)<EOL>excerpt_text = ELLIPSIS.join(matches)<EOL>for match_word in match_phrases:<EOL><INDENT>excerpt_text = SearchResultProcessor.decorate_matches(excerpt_text, match_word)<EOL><DEDENT>return excerpt_text<EOL>

Property to display a useful excerpt representing the matches within the results


@staticmethod<EOL><INDENT>def find_matches(strings, words, length_hoped):<DEDENT>

lower_words = [w.lower() for w in words]<EOL>def has_match(string):<EOL><INDENT>"""<STR_LIT>"""<EOL>lower_string = string.lower()<EOL>for test_word in lower_words:<EOL><INDENT>if test_word in lower_string:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL><DEDENT>shortened_strings = [textwrap.wrap(s) for s in strings]<EOL>short_string_list = list(chain.from_iterable(shortened_strings))<EOL>matches = [ms for ms in short_string_list if has_match(ms)]<EOL>cumulative_len = <NUM_LIT:0><EOL>break_at = None<EOL>for idx, match in enumerate(matches):<EOL><INDENT>cumulative_len += len(match)<EOL>if cumulative_len >= length_hoped:<EOL><INDENT>break_at = idx<EOL>break<EOL><DEDENT><DEDENT>return matches[<NUM_LIT:0>:break_at]<EOL>


strings = [value for value in six.itervalues(dictionary) if not isinstance(value, dict)]<EOL>for child_dict in [dv for dv in six.itervalues(dictionary) if isinstance(dv, dict)]:<EOL><INDENT>strings.extend(SearchResultProcessor.strings_in_dictionary(child_dict))<EOL><DEDENT>return strings<EOL>

Used by default implementation for finding excerpt


return False<EOL>

Override this in a class in order to add in last-chance access checks to the search process
Your application will want to make this decision


if encoding is None:<EOL><INDENT>encoding = b'<STR_LIT:utf-8>'<EOL><DEDENT>headers = _splitHeaders(<EOL>requestHeaders.getRawHeaders(b'<STR_LIT:Content-Type>', []))<EOL>if headers:<EOL><INDENT>return headers[<NUM_LIT:0>][<NUM_LIT:1>].get(b'<STR_LIT>', encoding)<EOL><DEDENT>return encoding<EOL>

Extract an encoding from a ``Content-Type`` header.

@type  requestHeaders: `twisted.web.http_headers.Headers`
@param requestHeaders: Request headers.

@type  encoding: `bytes`
@param encoding: Default encoding to assume if the ``Content-Type``
    header is lacking one. Defaults to ``UTF-8``.

@rtype: `bytes`
@return: Content encoding.


def sort(value):<EOL><INDENT>return float(value[<NUM_LIT:1>].get('<STR_LIT:q>', <NUM_LIT:1>))<EOL><DEDENT>return OrderedDict(sorted(_splitHeaders(headers), key=sort, reverse=True))<EOL>

Parse and sort an ``Accept`` header.

The header is sorted according to the ``q`` parameter for each header value.

@rtype: `OrderedDict` mapping `bytes` to `dict`
@return: Mapping of media types to header parameters.


return [cgi.parse_header(value)<EOL>for value in chain.from_iterable(<EOL>s.split('<STR_LIT:U+002C>') for s in headers<EOL>if s)]<EOL>

Split an HTTP header whose components are separated with commas.

Each component is then split on semicolons and the component arguments
converted into a `dict`.

@return: `list` of 2-`tuple` of `bytes`, `dict`
@return: List of header arguments and mapping of component argument names
    to values.


self._wrappedResource = wrappedResource<EOL>Resource.__init__(self)<EOL>

:type  wrappedResource: `ISpinneretResource`
:param wrappedResource: Spinneret resource to wrap in an `IResource`.


def __init__(self, handlers, fallback=False):

Resource.__init__(self)<EOL>self._handlers = list(handlers)<EOL>self._fallback = fallback<EOL>self._acceptHandlers = {}<EOL>for handler in self._handlers:<EOL><INDENT>for acceptType in handler.acceptTypes:<EOL><INDENT>if acceptType in self._acceptHandlers:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % (acceptType,))<EOL><DEDENT>self._acceptHandlers[acceptType] = handler<EOL><DEDENT><DEDENT>

:type  handlers: ``iterable`` of `INegotiableResource` and either
    `IResource` or `ISpinneretResource`.
:param handlers: Iterable of negotiable resources, either
    `ISpinneretResource` or `IResource`, to use as handlers for
    negotiation.

:type  fallback: `bool`
:param fallback: Fall back to the first handler in the case where
    negotiation fails?


Locate another object which can be adapted to `IResource`.

:type  request: `IRequest <twisted:twisted.web.iweb.IRequest>`
:param request: Request.

:type  segments: ``sequence`` of `bytes`
:param segments: Sequence of strings giving the remaining query
    segments to resolve.

:rtype: 2-`tuple` of `IResource`, `IRenderable` or `URLPath` and
    a ``sequence`` of `bytes`
:return: Pair of an `IResource`, `IRenderable` or `URLPath` and
    a sequence of the remaining path segments to be process, or
    a `Deferred` containing the aforementioned result.


try:<EOL><INDENT>return float(Text(value, encoding))<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>return None<EOL><DEDENT>

Parse a value as a floating point number.

:type  value: `unicode` or `bytes`
:param value: Text value to parse.

:type  encoding: `bytes`
:param encoding: Encoding to treat `bytes` values as, defaults to
    ``utf-8``.

:rtype: `float`
:return: Parsed float or ``None`` if ``value`` could not be parsed as a
    float.