parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>token_auth=True,<EOL>archive=True)<EOL>action = parser.parser._option_string_actions['<STR_LIT>']<EOL>action.required = True<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>type=int, default=MAX_ITEMS,<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>action='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>', default=TWEET_TYPE_MIXED,<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>default=MIN_RATE_LIMIT, type=int,<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>default=SLEEP_TIME, type=int,<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


def fetch_items(self, category, **kwargs):

since_id = kwargs['<STR_LIT>']<EOL>max_id = kwargs['<STR_LIT>']<EOL>geocode = kwargs['<STR_LIT>']<EOL>lang = kwargs['<STR_LIT>']<EOL>entities = kwargs['<STR_LIT>']<EOL>tweets_type = kwargs['<STR_LIT>']<EOL>logger.info("<STR_LIT>",<EOL>self.query, str(since_id),<EOL>str(max_id) if max_id else '<STR_LIT>')<EOL>tweets_ids = []<EOL>min_date = None<EOL>max_date = None<EOL>group_tweets = self.client.tweets(self.query, since_id=since_id, max_id=max_id, geocode=geocode,<EOL>lang=lang, include_entities=entities, result_type=tweets_type)<EOL>for tweets in group_tweets:<EOL><INDENT>for i in range(len(tweets)):<EOL><INDENT>tweet = tweets[i]<EOL>tweets_ids.append(tweet['<STR_LIT:id>'])<EOL>if tweets[-<NUM_LIT:1>] == tweet:<EOL><INDENT>min_date = str_to_datetime(tweets[-<NUM_LIT:1>]['<STR_LIT>'])<EOL><DEDENT>if tweets[<NUM_LIT:0>] == tweet and not max_date:<EOL><INDENT>max_date = str_to_datetime(tweets[<NUM_LIT:0>]['<STR_LIT>'])<EOL><DEDENT>yield tweet<EOL><DEDENT><DEDENT>logger.info("<STR_LIT>",<EOL>len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)<EOL>

Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


return CATEGORY_TWEET<EOL>

Extracts the category from a Twitter item.

        This backend only generates one type of item which is
        'tweet'.


return MeetupClient(self.api_token, self.max_items,<EOL>self.sleep_for_rate, self.min_rate_to_sleep, self.sleep_time,<EOL>self.archive, from_archive)<EOL>


return str(item['<STR_LIT:id>'])<EOL>

Extracts the identifier from a Meetup item.


@staticmethod<EOL><INDENT>def sanitize_for_archive(url, headers, payload):<DEDENT>

if MeetupClient.PKEY in payload:<EOL><INDENT>payload.pop(MeetupClient.PKEY)<EOL><DEDENT>if MeetupClient.PSIGN in payload:<EOL><INDENT>payload.pop(MeetupClient.PSIGN)<EOL><DEDENT>return url, headers, payload<EOL>

Sanitize payload of a HTTP request by removing the token information
        before storing/retrieving archived items

        :param: url: HTTP url request
        :param: headers: HTTP headers request
        :param: payload: HTTP payload request

        :returns url, headers and the sanitized payload


return True<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive


params = {<EOL>self.PKEY: self.api_key<EOL>}<EOL>response = self._call(self.POSTS, post_id,<EOL>params=params)<EOL>return response<EOL>

Retrieve the post whit `post_id` identifier.

        :param post_id: identifier of the post to retrieve


url = self.URL % {'<STR_LIT>': self.base_url, '<STR_LIT>': resource}<EOL>logger.debug("<STR_LIT>",<EOL>resource, str(params))<EOL>while True:<EOL><INDENT>r = self.fetch(url, payload=params)<EOL>yield r.text<EOL>j = r.json()<EOL>if '<STR_LIT>' not in j:<EOL><INDENT>break<EOL><DEDENT>if '<STR_LIT>' not in j['<STR_LIT>']:<EOL><INDENT>break<EOL><DEDENT>url = urijoin(self.base_url, j['<STR_LIT>']['<STR_LIT>'])<EOL>params = {}<EOL><DEDENT>

Retrive the given resource.

        :param resource: resource to retrieve
        :param params: dict with the HTTP parameters needed to retrieve
            the given resource


summary = json.loads(raw_json)<EOL>contents = summary['<STR_LIT>']<EOL>for c in contents:<EOL><INDENT>yield c<EOL><DEDENT>

Parse a Confluence summary JSON list.

        The method parses a JSON stream and returns an iterator
        of diccionaries. Each dictionary is a content summary.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed content summaries.


return ConfluenceClient(self.url, archive=self.archive, from_archive=from_archive)<EOL>


hc = json.loads(raw_json)<EOL>return hc<EOL>

Parse a Confluence historical content JSON stream.

        This method parses a JSON stream and returns a dictionary
        that contains the data of a historical content.

        :param raw_json: JSON string to parse

        :returns: a dict with historical content


def historical_content(self, content_id, version):

resource = self.RCONTENTS + '<STR_LIT:/>' + str(content_id)<EOL>params = {<EOL>self.PVERSION: version,<EOL>self.PSTATUS: self.VHISTORICAL,<EOL>self.PEXPAND: '<STR_LIT:U+002C>'.join(self.VEXPAND)<EOL>}<EOL>response = [response for response in self._call(resource, params)]<EOL>return response[<NUM_LIT:0>]<EOL>

Get the snapshot of a content for the given version.

        :param content_id: fetch the snapshot of this content
        :param version: snapshot version of the content


return CATEGORY_ARTICLE<EOL>

Extracts the category from a NNTP item.

        This backend only generates one type of item which is
        'article'.


return NNTTPClient(self.host, self.archive, from_archive)<EOL>


return True<EOL>

Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming


for phid in results['<STR_LIT:result>'].values():<EOL><INDENT>yield phid<EOL><DEDENT>

Parse a Phabicator PHIDs JSON stream.

        This method parses a JSON stream and returns a list iterator.
        Each item is a dictionary that contains the PHID parsed data.

        :param results: JSON to parse

        :returns: a generator of parsed PHIDs


<EOL>ts = int(datetime_to_utc(from_date).timestamp()) or <NUM_LIT:1><EOL>consts = {<EOL>self.PMODIFIED_START: ts<EOL>}<EOL>attachments = {<EOL>self. PPROJECTS: True<EOL>}<EOL>params = {<EOL>self.PCONSTRAINTS: consts,<EOL>self.PATTACHMENTS: attachments,<EOL>self.PORDER: self.VOUTDATED,<EOL>}<EOL>while True:<EOL><INDENT>r = self._call(self.MANIPHEST_TASKS, params)<EOL>yield r<EOL>j = json.loads(r)<EOL>after = j['<STR_LIT:result>']['<STR_LIT>']['<STR_LIT>']<EOL>if not after:<EOL><INDENT>break<EOL><DEDENT>params[self.PAFTER] = after<EOL><DEDENT>

Retrieve tasks.

        :param from_date: retrieve tasks that where updated from that date;
            dates are converted epoch time.


from_date = kwargs['<STR_LIT>']<EOL>logger.info("<STR_LIT>", self.url, str(from_date))<EOL>ntasks = <NUM_LIT:0><EOL>for task in self.__fetch_tasks(from_date):<EOL><INDENT>yield task<EOL>ntasks += <NUM_LIT:1><EOL><DEDENT>logger.info("<STR_LIT>", ntasks)<EOL>

Fetch the tasks

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


url = self.URL % {'<STR_LIT>': self.base_url, '<STR_LIT>': method}<EOL>params['<STR_LIT>'] = {'<STR_LIT>': self.api_token}<EOL>data = {<EOL>'<STR_LIT>': json.dumps(params, sort_keys=True),<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': True<EOL>}<EOL>logger.debug("<STR_LIT>",<EOL>method, str(data))<EOL>r = self.fetch(url, payload=data, method=HttpClient.POST, verify=False)<EOL>result = r.json()<EOL>if result['<STR_LIT>']:<EOL><INDENT>raise ConduitError(error=result['<STR_LIT>'],<EOL>code=result['<STR_LIT>'])<EOL><DEDENT>return r.text<EOL>

Call a method.

        :param method: method to call
        :param params: dict with the HTTP parameters needed to call
            the given method

        :raises ConduitError: when an error is returned by the server


return str(item['<STR_LIT:id>'])<EOL>

Extracts the identifier from a Phabricator item.


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>token_auth=True,<EOL>archive=True)<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>default=MAX_RETRIES, type=int,<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>default=DEFAULT_SLEEP_TIME, type=int,<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT:url>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>

Returns the Phabricator argument parser.


return CATEGORY_MESSAGE<EOL>

Extracts the category from a MBox item.

        This backend only generates one type of item which is
        'message'.


def _fetch_and_parse_messages(self, mailing_list, from_date):

from_date = datetime_to_utc(from_date)<EOL>nmsgs, imsgs, tmsgs = (<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>)<EOL>for mbox in mailing_list.mboxes:<EOL><INDENT>tmp_path = None<EOL>try:<EOL><INDENT>tmp_path = self._copy_mbox(mbox)<EOL>for message in self.parse_mbox(tmp_path):<EOL><INDENT>tmsgs += <NUM_LIT:1><EOL>if not self._validate_message(message):<EOL><INDENT>imsgs += <NUM_LIT:1><EOL>continue<EOL><DEDENT>dt = str_to_datetime(message[MBox.DATE_FIELD])<EOL>if dt < from_date:<EOL><INDENT>logger.debug("<STR_LIT>",<EOL>message['<STR_LIT>'], str(from_date))<EOL>tmsgs -= <NUM_LIT:1><EOL>continue<EOL><DEDENT>message = self._casedict_to_dict(message)<EOL>nmsgs += <NUM_LIT:1><EOL>logger.debug("<STR_LIT>", message['<STR_LIT>'])<EOL>yield message<EOL><DEDENT><DEDENT>except (OSError, EOFError) as e:<EOL><INDENT>logger.warning("<STR_LIT>", mbox.filepath, str(e))<EOL><DEDENT>except Exception as e:<EOL><INDENT>if tmp_path and os.path.exists(tmp_path):<EOL><INDENT>os.remove(tmp_path)<EOL><DEDENT>raise e<EOL><DEDENT>finally:<EOL><INDENT>if tmp_path and os.path.exists(tmp_path):<EOL><INDENT>os.remove(tmp_path)<EOL><DEDENT><DEDENT><DEDENT>logger.info("<STR_LIT>",<EOL>nmsgs, tmsgs, imsgs)<EOL>

Fetch and parse the messages from a mailing list


return True<EOL>


archives = []<EOL>if os.path.isfile(self.dirpath):<EOL><INDENT>try:<EOL><INDENT>archives.append(MBoxArchive(self.dirpath))<EOL><DEDENT>except OSError as e:<EOL><INDENT>logger.warning("<STR_LIT>", self.dirpath, str(e))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for root, _, files in os.walk(self.dirpath):<EOL><INDENT>for filename in sorted(files):<EOL><INDENT>try:<EOL><INDENT>location = os.path.join(root, filename)<EOL>archives.append(MBoxArchive(location))<EOL><DEDENT>except OSError as e:<EOL><INDENT>logger.warning("<STR_LIT>", filename, str(e))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return archives<EOL>

Get the mboxes managed by this mailing list.

        Returns the archives sorted by name.

        :returns: a list of `.MBoxArchive` objects


return False<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend does not support items archive


sanitized_cmd = re.sub(r"<STR_LIT>", '<STR_LIT>', cmd)<EOL>return sanitized_cmd<EOL>

Sanitize the Gerrit command by removing username information
        before storing/retrieving archived items

        :param: cmd: Gerrit command

        :returns the sanitized cmd


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>archive=True)<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT:user>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>type=int, default=MAX_REVIEWS,<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>nargs='<STR_LIT:*>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT:port>',<EOL>default=PORT, type=int,<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


return float(item['<STR_LIT>'])<EOL>

Extracts and converts the update time from a Gerrit item.

        The timestamp is extracted from 'lastUpdated' field. This date is
        a UNIX timestamp but needs to be converted to a float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


return True<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive


return CATEGORY_REVIEW<EOL>

Extracts the category from a Gerrit item.

        This backend only generates one type of item which is
        'review'.


<EOL>items_raw = "<STR_LIT:[>" + raw_data.replace("<STR_LIT:\n>", "<STR_LIT:U+002C>") + "<STR_LIT:]>"<EOL>items_raw = items_raw.replace("<STR_LIT>", "<STR_LIT:]>")<EOL>items = json.loads(items_raw)<EOL>reviews = []<EOL>for item in items:<EOL><INDENT>if '<STR_LIT>' in item.keys():<EOL><INDENT>reviews.append(item)<EOL><DEDENT><DEDENT>return reviews<EOL>


raw_questions = json.loads(raw_page)<EOL>questions = raw_questions['<STR_LIT>']<EOL>for question in questions:<EOL><INDENT>yield question<EOL><DEDENT>

Parse a StackExchange API raw response.

        The method parses the API response retrieving the
        questions from the received items

        :param items: items from where to parse the questions

        :returns: a generator of questions


return float(item['<STR_LIT>'])<EOL>

Extracts the update time from a StackExchange item.

        The timestamp is extracted from 'last_activity_date' field.
        This date is a UNIX timestamp but needs to be converted to
        a float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


ts = str_to_datetime(item['<STR_LIT>'])<EOL>return ts.timestamp()<EOL>

Extracts the update time from a RSS item.

        The timestamp is extracted from 'published' field.
        This date is a datetime string that needs to be converted to
        a UNIX timestamp float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


logger.info("<STR_LIT>", self.url)<EOL>nentries = <NUM_LIT:0>  <EOL>raw_entries = self.client.get_entries()<EOL>entries = self.parse_feed(raw_entries)['<STR_LIT>']<EOL>for item in entries:<EOL><INDENT>yield item<EOL>nentries += <NUM_LIT:1><EOL><DEDENT>logger.info("<STR_LIT>", nentries)<EOL>

Fetch the entries

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


return False<EOL>

Returns whether it supports to resume the fetch process.

        :returns: this backend does not supports entries resuming


params = {<EOL>"<STR_LIT:action>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>"<EOL>}<EOL>return self.call(params)<EOL>


from_date = kwargs['<STR_LIT>']<EOL>reviews_api = kwargs['<STR_LIT>']<EOL>mediawiki_version = self.client.get_version()<EOL>logger.info("<STR_LIT>", mediawiki_version)<EOL>if reviews_api:<EOL><INDENT>if ((mediawiki_version[<NUM_LIT:0>] == <NUM_LIT:1> and mediawiki_version[<NUM_LIT:1>] >= <NUM_LIT>) or mediawiki_version[<NUM_LIT:0>] > <NUM_LIT:1>):<EOL><INDENT>fetcher = self.__fetch_1_27(from_date)<EOL><DEDENT>else:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL>logger.warning("<STR_LIT>")<EOL>fetcher = self.__fetch_pre1_27(from_date)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fetcher = self.__fetch_pre1_27(from_date)<EOL><DEDENT>for page_reviews in fetcher:<EOL><INDENT>yield page_reviews<EOL><DEDENT>

Fetch the pages

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>archive=True)<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT:url>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


logger.debug("<STR_LIT>",<EOL>self.base_url, str(params))<EOL>req = self.fetch(self.base_url, payload=params)<EOL>return req.text<EOL>

Run an API command.
        :param cgi: cgi command to run on the server
        :param params: dict with the HTTP parameters needed to run
            the given command


return CATEGORY_PAGE<EOL>

Extracts the category from a MediaWiki item.

        This backend only generates one type of item which is
        'page'.


return CATEGORY_MESSAGE<EOL>

Extracts the category from a Slack item.

        This backend only generates one type of item which is
        'message'.


resource = self.RUSER_INFO<EOL>params = {<EOL>self.PUSER: user_id<EOL>}<EOL>response = self._fetch(resource, params)<EOL>return response<EOL>


resource = self.RCHANNEL_INFO<EOL>params = {<EOL>self.PCHANNEL: channel,<EOL>}<EOL>response = self._fetch(resource, params)<EOL>return response<EOL>


return True<EOL>


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>token_auth=True,<EOL>archive=True)<EOL>action = parser.parser._option_string_actions['<STR_LIT>']<EOL>action.required = True<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>type=int, default=MAX_ITEMS,<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


result = json.loads(raw_channel_info)<EOL>return result['<STR_LIT>']<EOL>

Parse a channel info JSON stream.

        This method parses a JSON stream, containing the information
        from a channel, and returns a dict with the parsed data.

        :param raw_channel_info

        :returns: a dict with the parsed information about a channel


result = json.loads(raw_user)<EOL>return result['<STR_LIT:user>']<EOL>

Parse a user's info JSON stream.

        This method parses a JSON stream, containing the information
        from a user, and returns a dict with the parsed data.

        :param raw_user: JSON string to parse

        :returns: a dict with the parsed user's information


return CATEGORY_BUILD<EOL>

Extracts the category from a Jenkins item.

        This backend only generates one type of item which is
        'build'.


return str(item['<STR_LIT:url>'])<EOL>

Extracts the identifier from a Build item.


patterns = [(self.SUPYBOT_COMMENT_REGEX, self.TCOMMENT),<EOL>(self.SUPYBOT_COMMENT_ACTION_REGEX, self.TCOMMENT),<EOL>(self.SUPYBOT_SERVER_REGEX, self.TSERVER),<EOL>(self.SUPYBOT_BOT_REGEX, self.TCOMMENT)]<EOL>for p in patterns:<EOL><INDENT>m = p[<NUM_LIT:0>].match(line)<EOL>if not m:<EOL><INDENT>continue<EOL><DEDENT>return p[<NUM_LIT:1>], m.group('<STR_LIT>'), m.group('<STR_LIT:body>').strip()<EOL><DEDENT>msg = "<STR_LIT>" % (str(self.nline))<EOL>raise ParseError(cause=msg)<EOL>


with open(filepath, '<STR_LIT:r>', errors='<STR_LIT>',<EOL>newline=os.linesep) as f:<EOL><INDENT>parser = SupybotParser(f)<EOL>try:<EOL><INDENT>for message in parser.parse():<EOL><INDENT>yield message<EOL><DEDENT><DEDENT>except ParseError as e:<EOL><INDENT>cause = "<STR_LIT>" % (filepath, str(e))<EOL>raise ParseError(cause=cause)<EOL><DEDENT><DEDENT>

Parse a Supybot IRC log file.

        The method parses the Supybot IRC log file and returns an iterator of
        dictionaries. Each one of this, contains a message from the file.

        :param filepath: path to the IRC log file

        :returns: a generator of parsed messages

        :raises ParseError: raised when the format of the Supybot log file
            is invalid
        :raises OSError: raised when an error occurs reading the
            given file


archives = []<EOL>candidates = self.__list_supybot_archives()<EOL>for candidate in candidates:<EOL><INDENT>dt = self.__parse_date_from_filepath(candidate)<EOL>if dt.date() >= from_date.date():<EOL><INDENT>archives.append((dt, candidate))<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>",<EOL>candidate, str(from_date))<EOL><DEDENT><DEDENT>archives.sort(key=lambda x: x[<NUM_LIT:0>])<EOL>return [archive[<NUM_LIT:1>] for archive in archives]<EOL>

Retrieve the Supybot archives after the given date


return True<EOL>

Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming


aliases = {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>aliases=aliases)<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


ts = item['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>']<EOL>ts = str_to_datetime(ts)<EOL>ts = ts.replace(tzinfo=dateutil.tz.tzutc())<EOL>return ts.timestamp()<EOL>

Extracts and coverts the update time from a Bugzilla item.

        The timestamp is extracted from 'delta_ts' field. This date is
        converted to UNIX timestamp format. Due Bugzilla servers ignore
        the timezone on HTTP requests, it will be ignored during the
        conversion, too.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


return CATEGORY_BUG<EOL>

Extracts the category from a Bugzilla item.

        This backend only generates one type of item which is
        'bug'.


def fetch(self, category=CATEGORY_BUG, from_date=DEFAULT_DATETIME):

if not from_date:<EOL><INDENT>from_date = DEFAULT_DATETIME<EOL><DEDENT>kwargs = {"<STR_LIT>": from_date}<EOL>items = super().fetch(category, **kwargs)<EOL>return items<EOL>

Fetch the bugs from the repository.

        The method retrieves, from a Bugzilla repository, the bugs
        updated since the given date.

        :param category: the category of items to fetch
        :param from_date: obtain bugs updated since this date

        :returns: a generator of bugs


def is_activity_empty(bs):<EOL><INDENT>EMPTY_ACTIVITY = "<STR_LIT>"<EOL>tag = bs.find(text=re.compile(EMPTY_ACTIVITY))<EOL>return tag is not None<EOL><DEDENT>def find_activity_table(bs):<EOL><INDENT>tables = bs.find_all('<STR_LIT>')<EOL>for tb in tables:<EOL><INDENT>nheaders = len(tb.tr.find_all('<STR_LIT>', recursive=False))<EOL>if nheaders == <NUM_LIT:5>:<EOL><INDENT>return tb<EOL><DEDENT><DEDENT>raise ParseError(cause="<STR_LIT>")<EOL><DEDENT>def remove_tags(bs):<EOL><INDENT>HTML_TAGS_TO_REMOVE = ['<STR_LIT:a>', '<STR_LIT:i>', '<STR_LIT>']<EOL>for tag in bs.find_all(HTML_TAGS_TO_REMOVE):<EOL><INDENT>tag.replaceWith(tag.text)<EOL><DEDENT><DEDENT>def format_text(bs):<EOL><INDENT>strings = [s.strip('<STR_LIT>') for s in bs.stripped_strings]<EOL>s = '<STR_LIT:U+0020>'.join(strings)<EOL>return s<EOL><DEDENT>bs = bs4.BeautifulSoup(raw_html, '<STR_LIT>')<EOL>if is_activity_empty(bs):<EOL><INDENT>fields = []<EOL><DEDENT>else:<EOL><INDENT>activity_tb = find_activity_table(bs)<EOL>remove_tags(activity_tb)<EOL>fields = activity_tb.find_all('<STR_LIT>')<EOL><DEDENT>while fields:<EOL><INDENT>who = fields.pop(<NUM_LIT:0>)<EOL>when = fields.pop(<NUM_LIT:0>)<EOL>n = int(who.get('<STR_LIT>'))<EOL>for _ in range(n):<EOL><INDENT>what = fields.pop(<NUM_LIT:0>)<EOL>removed = fields.pop(<NUM_LIT:0>)<EOL>added = fields.pop(<NUM_LIT:0>)<EOL>event = {'<STR_LIT>': format_text(who),<EOL>'<STR_LIT>': format_text(when),<EOL>'<STR_LIT>': format_text(what),<EOL>'<STR_LIT>': format_text(removed),<EOL>'<STR_LIT>': format_text(added)}<EOL>yield event<EOL><DEDENT><DEDENT>

Parse a Bugzilla bug activity HTML stream.

        This method extracts the information about activity from the
        given HTML stream. The bug activity is stored into a HTML
        table. Each parsed activity event is returned into a dictionary.

        If the given HTML is invalid, the method will raise a ParseError
        exception.

        :param raw_html: HTML string to parse

        :returns: a generator of parsed activity events

        :raises ParseError: raised when an error occurs parsing
            the given HTML stream


def __fetch_user_data(self, tag_type, user_link):

user_name = self.client.user_name(user_link)<EOL>user = {}<EOL>if not user_name:<EOL><INDENT>return user<EOL><DEDENT>user_raw = self.client.user(user_name)<EOL>user = json.loads(user_raw)<EOL>return user<EOL>


path = urijoin("<STR_LIT>", str(issue_id))<EOL>url_issue = self.__get_url(path)<EOL>raw_text = self.__send_request(url_issue)<EOL>return raw_text<EOL>


return CATEGORY_ISSUE<EOL>

Extracts the category from a Launchpad item.

        This backend only generates one type of item which is
        'issue'.


return True<EOL>


issue['<STR_LIT>'] = {}<EOL>issue['<STR_LIT>'] = {}<EOL>issue['<STR_LIT>'] = {}<EOL>return issue<EOL>


for messages_raw in self.client.issue_collection(issue_id, "<STR_LIT>"):<EOL><INDENT>messages = json.loads(messages_raw)<EOL>for msg in messages['<STR_LIT>']:<EOL><INDENT>msg['<STR_LIT>'] = self.__fetch_user_data('<STR_LIT>', msg['<STR_LIT>'])<EOL>yield msg<EOL><DEDENT><DEDENT>


raw_issue = self.client.issue(issue_id)<EOL>issue = json.loads(raw_issue)<EOL>return issue<EOL>


from_date = kwargs['<STR_LIT>']<EOL>logger.info("<STR_LIT>",<EOL>self.distribution, str(from_date))<EOL>nissues = <NUM_LIT:0><EOL>for issue in self._fetch_issues(from_date):<EOL><INDENT>yield issue<EOL>nissues += <NUM_LIT:1><EOL><DEDENT>logger.info("<STR_LIT>", nissues)<EOL>

Fetch the issues

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


return False<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend does not support items archive


raw_issues = json.loads(raw_page)<EOL>issues = raw_issues['<STR_LIT>']<EOL>for issue in issues:<EOL><INDENT>yield issue<EOL><DEDENT>

Parse a JIRA API raw response.

        The method parses the API response retrieving the
        issues from the received items

        :param items: items from where to parse the issues

        :returns: a generator of issues


def fetch(self, category=CATEGORY_ISSUE, from_date=DEFAULT_DATETIME):

if not from_date:<EOL><INDENT>from_date = DEFAULT_DATETIME<EOL><DEDENT>from_date = datetime_to_utc(from_date)<EOL>kwargs = {'<STR_LIT>': from_date}<EOL>items = super().fetch(category, **kwargs)<EOL>return items<EOL>

Fetch the issues from the site.

        The method retrieves, from a JIRA site, the
        issues updated since the given date.

        :param category: the category of items to fetch
        :param from_date: retrieve issues updated from this date

        :returns: a generator of issues


url = urijoin(self.base_url, self.RESOURCE, self.VERSION_API, '<STR_LIT>')<EOL>req = self.fetch(url)<EOL>return req.text<EOL>


from_date = kwargs['<STR_LIT>']<EOL>logger.info("<STR_LIT>",<EOL>self.url, self.project, str(from_date))<EOL>whole_pages = self.client.get_issues(from_date)<EOL>fields = json.loads(self.client.get_fields())<EOL>custom_fields = filter_custom_fields(fields)<EOL>for whole_page in whole_pages:<EOL><INDENT>issues = self.parse_issues(whole_page)<EOL>for issue in issues:<EOL><INDENT>mapping = map_custom_field(custom_fields, issue['<STR_LIT>'])<EOL>for k, v in mapping.items():<EOL><INDENT>issue['<STR_LIT>'][k] = v<EOL><DEDENT>comments_data = self.__get_issue_comments(issue['<STR_LIT:id>'])<EOL>issue['<STR_LIT>'] = comments_data<EOL>yield issue<EOL><DEDENT><DEDENT>


def get_items(self, from_date, url, expand_fields=True):

start_at = <NUM_LIT:0><EOL>req = self.fetch(url, payload=self.__build_payload(start_at, from_date, expand_fields))<EOL>issues = req.text<EOL>data = req.json()<EOL>titems = data['<STR_LIT>']<EOL>nitems = data['<STR_LIT>']<EOL>start_at += min(nitems, titems)<EOL>self.__log_status(start_at, titems, url)<EOL>while issues:<EOL><INDENT>yield issues<EOL>issues = None<EOL>if data['<STR_LIT>'] + nitems < titems:<EOL><INDENT>req = self.fetch(url, payload=self.__build_payload(start_at, from_date, expand_fields))<EOL>data = req.json()<EOL>start_at += nitems<EOL>issues = req.text<EOL>self.__log_status(start_at, titems, url)<EOL><DEDENT><DEDENT>

Retrieve all the items from a given date.

        :param url: endpoint API url
        :param from_date: obtain items updated since this date
        :param expand_fields: if True, it includes the expand fields in the payload


return CATEGORY_ISSUE<EOL>

Extracts the category from a Jira item.

        This backend only generates one type of item which is
        'issue'.


return JiraClient(self.url, self.project, self.user, self.password,<EOL>self.verify, self.cert, self.max_results,<EOL>self.archive, from_archive)<EOL>


issues_groups = self.client.issues(from_date=from_date)<EOL>for raw_issues in issues_groups:<EOL><INDENT>issues = json.loads(raw_issues)<EOL>for issue in issues:<EOL><INDENT>issue_id = issue['<STR_LIT>']<EOL>if self.blacklist_ids and issue_id in self.blacklist_ids:<EOL><INDENT>logger.warning("<STR_LIT>", issue_id)<EOL>continue<EOL><DEDENT>self.__init_issue_extra_fields(issue)<EOL>issue['<STR_LIT>'] =self.__get_issue_notes(issue_id)<EOL>issue['<STR_LIT>'] =self.__get_award_emoji(GitLabClient.ISSUES, issue_id)<EOL>yield issue<EOL><DEDENT><DEDENT>


page = <NUM_LIT:0>  <EOL>last_page = None  <EOL>url_next = urijoin(self.base_url, GitLabClient.PROJECTS, self.owner + '<STR_LIT>' + self.repository, path)<EOL>logger.debug("<STR_LIT>" + url_next)<EOL>response = self.fetch(url_next, payload=payload)<EOL>items = response.text<EOL>page += <NUM_LIT:1><EOL>if '<STR_LIT>' in response.links:<EOL><INDENT>last_url = response.links['<STR_LIT>']['<STR_LIT:url>']<EOL>last_page = last_url.split('<STR_LIT>')[<NUM_LIT:1>].split('<STR_LIT:&>')[<NUM_LIT:0>]<EOL>last_page = int(last_page)<EOL>logger.debug("<STR_LIT>" % (page, last_page))<EOL><DEDENT>while items:<EOL><INDENT>yield items<EOL>items = None<EOL>if '<STR_LIT>' in response.links:<EOL><INDENT>url_next = response.links['<STR_LIT>']['<STR_LIT:url>']  <EOL>response = self.fetch(url_next, payload=payload)<EOL>page += <NUM_LIT:1><EOL>items = response.text<EOL>logger.debug("<STR_LIT>" % (page, last_page))<EOL><DEDENT><DEDENT>

Return the items from GitLab API using links pagination


return GitLabClient(self.owner, self.repository, self.api_token, self.base_url,<EOL>self.sleep_for_rate, self.min_rate_to_sleep,<EOL>self.sleep_time, self.max_retries,<EOL>self.archive, from_archive)<EOL>


def merge_version(self, merge_id, version_id):

path = urijoin(self.base_url,<EOL>GitLabClient.PROJECTS, self.owner + '<STR_LIT>' + self.repository,<EOL>GitLabClient.MERGES, merge_id, GitLabClient.VERSIONS, version_id)<EOL>response = self.fetch(path)<EOL>return response.text<EOL>


def note_emojis(self, item_type, item_id, note_id):

payload = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': PER_PAGE<EOL>}<EOL>path = urijoin(item_type, str(item_id), GitLabClient.NOTES,<EOL>str(note_id), GitLabClient.EMOJI)<EOL>return self.fetch_items(path, payload)<EOL>


def __get_award_emoji(self, item_type, item_id):

emojis = []<EOL>group_emojis = self.client.emojis(item_type, item_id)<EOL>for raw_emojis in group_emojis:<EOL><INDENT>for emoji in json.loads(raw_emojis):<EOL><INDENT>emojis.append(emoji)<EOL><DEDENT><DEDENT>return emojis<EOL>

Get award emojis for issue/merge request


versions = []<EOL>group_versions = self.client.merge_versions(merge_id)<EOL>for raw_versions in group_versions:<EOL><INDENT>for version in json.loads(raw_versions):<EOL><INDENT>version_id = version['<STR_LIT:id>']<EOL>version_full_raw = self.client.merge_version(merge_id, version_id)<EOL>version_full = json.loads(version_full_raw)<EOL>version_full.pop('<STR_LIT>', None)<EOL>versions.append(version_full)<EOL><DEDENT><DEDENT>return versions<EOL>


path = urijoin(self.base_url,<EOL>GitLabClient.PROJECTS, self.owner + '<STR_LIT>' + self.repository,<EOL>GitLabClient.MERGES, merge_id)<EOL>response = self.fetch(path)<EOL>return response.text<EOL>


Fetch the issues/merge requests from the repository.

        The method retrieves, from a GitLab repository, the issues/merge requests
        updated since the given date.

        :param category: the category of items to fetch
        :param from_date: obtain issues updated since this date

        :returns: a generator of issues


notes = []<EOL>group_notes = self.client.notes(GitLabClient.ISSUES, issue_id)<EOL>for raw_notes in group_notes:<EOL><INDENT>for note in json.loads(raw_notes):<EOL><INDENT>note_id = note['<STR_LIT:id>']<EOL>note['<STR_LIT>'] =self.__get_note_award_emoji(GitLabClient.ISSUES, issue_id, note_id)<EOL>notes.append(note)<EOL><DEDENT><DEDENT>return notes<EOL>


def _set_auth_arguments(self, basic_auth=True, token_auth=False):

group = self.parser.add_argument_group('<STR_LIT>')<EOL>if basic_auth:<EOL><INDENT>group.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT:user>',<EOL>help="<STR_LIT>")<EOL>group.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT:password>',<EOL>help="<STR_LIT>")<EOL><DEDENT>if token_auth:<EOL><INDENT>group.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL><DEDENT>

Activate authentication arguments parsing


def check_value(v):<EOL><INDENT>if not isinstance(v, str):<EOL><INDENT>raise ValueError("<STR_LIT>" % str(v))<EOL><DEDENT>elif not v:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return v<EOL><DEDENT><DEDENT>s = '<STR_LIT::>'.join(map(check_value, args))<EOL>sha1 = hashlib.sha1(s.encode('<STR_LIT:utf-8>', errors='<STR_LIT>'))<EOL>uuid_sha1 = sha1.hexdigest()<EOL>return uuid_sha1<EOL>

Generate a UUID based on the given parameters.

    The UUID will be the SHA1 of the concatenation of the values
    from the list. The separator bewteedn these values is ':'.
    Each value must be a non-empty string, otherwise, the function
    will raise an exception.

    :param *args: list of arguments used to generate the UUID

    :returns: a universal unique identifier

    :raises ValueError: when anyone of the values is not a string,
        is empty or `None`.


pass<EOL>

Override to execute after backend is initialized.


if not self.archive:<EOL><INDENT>raise ArchiveError(cause="<STR_LIT>")<EOL><DEDENT>self.client = self._init_client(from_archive=True)<EOL>for item in self.fetch_items(self.archive.category, **self.archive.backend_params):<EOL><INDENT>yield self.metadata(item)<EOL><DEDENT>

Fetch the questions from an archive.

        It returns the items stored within an archive. If this method is called but
        no archive was provided, the method will raise a `ArchiveError` exception.

        :returns: a generator of items

        :raises ArchiveError: raised when an error occurs accessing an archive


def fetch(self, category, filter_classified=False, **kwargs):

if category not in self.categories:<EOL><INDENT>cause = "<STR_LIT>" % (category, self.__class__.__name__)<EOL>raise BackendError(cause=cause)<EOL><DEDENT>if filter_classified and self.archive:<EOL><INDENT>cause = "<STR_LIT>"<EOL>raise BackendError(cause=cause)<EOL><DEDENT>if self.archive:<EOL><INDENT>self.archive.init_metadata(self.origin, self.__class__.__name__, self.version, category,<EOL>kwargs)<EOL><DEDENT>self.client = self._init_client()<EOL>for item in self.fetch_items(category, **kwargs):<EOL><INDENT>if filter_classified:<EOL><INDENT>item = self.filter_classified_data(item)<EOL><DEDENT>yield self.metadata(item, filter_classified=filter_classified)<EOL><DEDENT>

Fetch items from the repository.

        The method retrieves items from a repository.

        To removed classified fields from the resulting items, set
        the parameter `filter_classified`. Take into account this
        parameter is incompatible with archiving items. Raw client
        data are archived before any other process. Therefore,
        classified data  are stored within the archive. To prevent
        from possible data leaks or security issues when users do
        not need these fields, archiving and filtering are not
        compatible.

        :param category: the category of the items fetched
        :param filter_classified: remove classified fields from the resulting items
        :param kwargs: a list of other parameters (e.g., from_date, offset, etc.
        specific for each backend)

        :returns: a generator of items

        :raises BackendError: either when the category is not valid or
            'filter_classified' and 'archive' are active at the same time.


restore_op_name = "<STR_LIT>"<EOL>filename_tensor_name = "<STR_LIT>"<EOL>input_graph_def = graph_pb2.GraphDef()<EOL>mode = "<STR_LIT:r>"<EOL>with gfile.FastGFile(input_graph, mode) as f:<EOL><INDENT>text_format.Merge(f.read().decode("<STR_LIT:utf-8>"), input_graph_def)<EOL><DEDENT>for node in input_graph_def.node:<EOL><INDENT>node.device = "<STR_LIT>"<EOL><DEDENT>_ = importer.import_graph_def(input_graph_def, name="<STR_LIT>")<EOL>with session.Session() as sess:<EOL><INDENT>sess.run([restore_op_name], {filename_tensor_name: input_checkpoint})<EOL>output_graph_def = graph_util.convert_variables_to_constants(<EOL>sess,<EOL>input_graph_def,<EOL>output_node_names,<EOL>variable_names_blacklist="<STR_LIT>")<EOL><DEDENT>with gfile.GFile(output_graph, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write(output_graph_def.SerializeToString())<EOL><DEDENT>

merge the checkpoint file with the non-binary graph file to
generate one GraphDef file with the variable values
Args:
    input_graph: the GraphDef file, not in the binary form
    input_checkpoint: the checkpoint file
    output_node_names: A string of name of the output names,
        use comma to seperate multi outputs
    output_graph: String of the location and the name of the
        output graph


height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>with slim.arg_scope(overfeat.overfeat_arg_scope()):<EOL><INDENT>net, end_points = overfeat.overfeat(inputs, is_training = False)<EOL><DEDENT>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>

You can also run these commands manually to generate the pb file
1. git clone https://github.com/tensorflow/models.git
2. export PYTHONPATH=Path_to_your_model_folder
3. python alexnet.py


tf.set_random_seed(<NUM_LIT:1>)<EOL>height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:2>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points  = vgg.vgg_19(inputs, is_training = False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


tf.set_random_seed(<NUM_LIT>)<EOL>input_width = <NUM_LIT:32><EOL>input_channel = <NUM_LIT:3><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, input_width, input_channel)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>filter_width = <NUM_LIT:4><EOL>output_channels = <NUM_LIT:6><EOL>filters = tf.Variable(tf.random_uniform((filter_width, input_channel, output_channels)))<EOL>conv_out = tf.nn.conv1d(inputs, filters, stride=<NUM_LIT:1>, padding="<STR_LIT>")<EOL>bias = tf.Variable(tf.zeros([output_channels]))<EOL>output = tf.nn.tanh(tf.nn.bias_add(conv_out, bias), name="<STR_LIT>")<EOL>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], backward=(argv[<NUM_LIT:3>] == '<STR_LIT:True>'))<EOL>

You can also run these commands manually to generate the pb file
1. git clone https://github.com/tensorflow/models.git
2. export PYTHONPATH=Path_to_your_model_folder
3. python temporal_convolution.py


height, width = <NUM_LIT>, <NUM_LIT><EOL>batchSize = <NUM_LIT:1><EOL>if len(argv) == <NUM_LIT:5>:<EOL><INDENT>batchSize = int(argv[<NUM_LIT:4>])<EOL><DEDENT>inputs = tf.Variable(tf.random_uniform((batchSize, <NUM_LIT:3>, height, width)), name = '<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points = alexnet_v1(inputs, is_training=False, spatial_squeeze=False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


print('<STR_LIT>', f.name)<EOL>with gzip.GzipFile(fileobj=f) as bytestream:<EOL><INDENT>magic = _read32(bytestream)<EOL>if magic != <NUM_LIT>:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' %<EOL>(magic, f.name))<EOL><DEDENT>num_images = _read32(bytestream)<EOL>rows = _read32(bytestream)<EOL>cols = _read32(bytestream)<EOL>buf = bytestream.read(rows * cols * num_images)<EOL>data = numpy.frombuffer(buf, dtype=numpy.uint8)<EOL>data = data.reshape(num_images, rows, cols, <NUM_LIT:1>)<EOL>return data<EOL><DEDENT>

Extract the images into a 4D uint8 numpy array [index, y, x, depth].

    :param: f: A file object that can be passed into a gzip reader.
    :return: data: A 4D unit8 numpy array [index, y, x, depth].
    :raise: ValueError: If the bytestream does not start with 2051.


return self.getOrDefault(self.maxEpoch)<EOL>

Gets the value of maxEpoch or its default value.


self._transfer_params_to_java()<EOL>return callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, dataset)<EOL>

Apply the transformer to the images in "inputCol" and store the transformed result
into "outputCols"


X, Y = mnist.read_data_sets(location, data_type)<EOL>return X, Y + <NUM_LIT:1><EOL>

Get mnist dataset with features and label as ndarray.
Data would be downloaded automatically if it doesn't present at the specific location.

:param data_type: "train" for training data and "test" for testing data.
:param location: Location to store mnist dataset.
:return: (features: ndarray, label: ndarray)


def __init__(self, sc=None, layer=None, pickle_registry=None, path=None, bigdl_type="<STR_LIT:float>"):

if layer is not None:<EOL><INDENT>self.bigdl_type = layer.bigdl_type<EOL><DEDENT>else:<EOL><INDENT>self.bigdl_type = bigdl_type<EOL><DEDENT>super(ModelBroadcast, self).__init__(sc, layer, pickle_registry, path)<EOL>

Should not be called directly by users -- use L{SparkContext.broadcast()}
instead.


return self.getOrDefault(self.batchSize)<EOL>

Gets the value of batchSize or its default value.


@staticmethod<EOL><INDENT>def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False):<DEDENT>

local_file_path = BCommon.get_local_file(filepath)<EOL>kmodel.load_weights(filepath=local_file_path, by_name=by_name)<EOL>WeightLoader.load_weights_from_kmodel(bmodel, kmodel)<EOL>

Loads all layer weights from a HDF5 save file.
        filepath can be stored in a local file system, HDFS, S3,
        or any Hadoop-supported file system.
        If `by_name` is False (default) weights are loaded
        based on the network's execution order topology,
        meaning layers in the execution seq should be exactly the same
        the architecture

        If `by_name` is True, weights are loaded into layers
        only if they share the same name. This is useful
        for fine-tuning or transfer-learning models where
        some of the layers have changed.


from keras.models import load_model<EOL>hdf5_local_path = BCommon.get_local_file(hdf5_path)<EOL>kmodel = load_model(hdf5_local_path)<EOL>return kmodel, DefinitionLoader.from_kmodel(kmodel)<EOL>

:param hdf5_path: hdf5 path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system.
:return: BigDL Model


node_id_to_config_layer[kmodel.name] = kmodel  <EOL>def gather_result(layers):<EOL><INDENT>if layers:  <EOL><INDENT>for layer in layers:<EOL><INDENT>if layer.name not in node_id_to_config_layer:<EOL><INDENT>node_id_to_config_layer[layer.name] = layer<EOL>DefinitionLoader.__build_node_id_2_klayer(layer, node_id_to_config_layer)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if hasattr(kmodel, "<STR_LIT>"):<EOL><INDENT>gather_result(kmodel.layers)<EOL><DEDENT>if hasattr(kmodel, "<STR_LIT>"):<EOL><INDENT>gather_result(kmodel.flattened_layers)<EOL><DEDENT>

The result would contain all of the layers including nested layers.
:param kmodel: a keras model which can be Sequential or Model
:param node_id_to_config_layer: a container to store the result


import pyspark<EOL>if(hasattr(pyspark,"<STR_LIT:version>")):<EOL><INDENT>full_version = pyspark.version.__version__<EOL>parts = full_version.split("<STR_LIT:.>")<EOL>spark_version = parts[<NUM_LIT:0>] + "<STR_LIT:.>" + parts[<NUM_LIT:1>]<EOL>if(compare_version(spark_version, "<STR_LIT>")>=<NUM_LIT:0>):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>


reader = tf.train.NewCheckpointReader(checkpoint_path)<EOL>tensor_names = filter(lambda n: n!='<STR_LIT>',<EOL>reader.get_variable_to_shape_map().keys())<EOL>tensors = {}<EOL>for tn in tensor_names:<EOL><INDENT>tensors[tn] = reader.get_tensor(tn)<EOL><DEDENT>return tensors<EOL>

Export variable tensors from the checkpoint files.

:param checkpoint_path: tensorflow checkpoint path
:return: dictionary of tensor. The key is the variable name and the value is the numpy


restore_op_name = "<STR_LIT>"<EOL>filename_tensor_name = "<STR_LIT>"<EOL>input_graph_def = graph_pb2.GraphDef()<EOL>with gfile.FastGFile(input_graph, "<STR_LIT:r>") as f:<EOL><INDENT>text_format.Merge(f.read().decode("<STR_LIT:utf-8>"), input_graph_def)<EOL><DEDENT>for node in input_graph_def.node:<EOL><INDENT>node.device = "<STR_LIT>"<EOL><DEDENT>importer.import_graph_def(input_graph_def, name="<STR_LIT>")<EOL>sess.run([restore_op_name], {filename_tensor_name: checkpoint})<EOL>output_graph_def = graph_util.convert_variables_to_constants(<EOL>sess,<EOL>input_graph_def,<EOL>output_node_names,<EOL>variable_names_blacklist="<STR_LIT>"<EOL>)<EOL>with gfile.GFile(output_graph, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write(output_graph_def.SerializeToString())<EOL><DEDENT>

Get the variable values from the checkpoint file, and merge them to the GraphDef file
Args:
    input_graph: the GraphDef file, doesn't contain variable values
    checkpoint: the checkpoint file
    output_node_names: A list of string, the output names
    output_graph: String of the location and the name of the
        output graph


if hasattr(SparkContext, "<STR_LIT>"):<EOL><INDENT>with SparkContext._lock:<EOL><INDENT>if SparkContext._active_spark_context is None:<EOL><INDENT>spark_conf = create_spark_conf() if conf is None else conf<EOL>return SparkContext.getOrCreate(spark_conf)<EOL><DEDENT>else:<EOL><INDENT>return SparkContext.getOrCreate()<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if SparkContext._active_spark_context is None:<EOL><INDENT>spark_conf = create_spark_conf() if conf is None else conf<EOL>return SparkContext(conf=spark_conf)<EOL><DEDENT>else:<EOL><INDENT>return SparkContext._active_spark_context<EOL><DEDENT><DEDENT>

Get the current active spark context and create one if no active instance
:param conf: combining bigdl configs into spark conf
:return: SparkContext


if isinstance(obj, RDD):<EOL><INDENT>obj = _to_java_object_rdd(obj)<EOL><DEDENT>elif isinstance(obj, DataFrame):<EOL><INDENT>obj = obj._jdf<EOL><DEDENT>elif isinstance(obj, SparkContext):<EOL><INDENT>obj = obj._jsc<EOL><DEDENT>elif isinstance(obj, (list, tuple)):<EOL><INDENT>obj = ListConverter().convert([_py2java(gateway, x) for x in obj],<EOL>gateway._gateway_client)<EOL><DEDENT>elif isinstance(obj, dict):<EOL><INDENT>result = {}<EOL>for (key, value) in obj.items():<EOL><INDENT>result[key] = _py2java(gateway, value)<EOL><DEDENT>obj = MapConverter().convert(result, gateway._gateway_client)<EOL><DEDENT>elif isinstance(obj, JavaValue):<EOL><INDENT>obj = obj.value<EOL><DEDENT>elif isinstance(obj, JavaObject):<EOL><INDENT>pass<EOL><DEDENT>elif isinstance(obj, (int, long, float, bool, bytes, unicode)):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>data = bytearray(PickleSerializer().dumps(obj))<EOL>obj = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data)<EOL><DEDENT>return obj<EOL>


@classmethod<EOL><INDENT>def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="<STR_LIT:float>"):<DEDENT>

if a_ndarray is None:<EOL><INDENT>return None<EOL><DEDENT>assert isinstance(a_ndarray, np.ndarray),"<STR_LIT>" % type(a_ndarray)<EOL>assert isinstance(i_ndarray, np.ndarray),"<STR_LIT>" % type(a_ndarray)<EOL>assert i_ndarray.size == a_ndarray.size * shape.size,"<STR_LIT>"<EOL>return cls(a_ndarray,<EOL>shape,<EOL>bigdl_type,<EOL>i_ndarray)<EOL>

Convert a three ndarray to SparseTensor which would be used in Java side.
For example:
a_ndarray = [1, 3, 2, 4]
i_ndarray = [[0, 0, 1, 2],
             [0, 3, 2, 1]]
shape = [3, 4]
Present a dense tensor
[[ 1,  0,  0,  3],
 [ 0,  0,  2,  0],
 [ 0,  4,  0,  0]]

:param a_ndarray non-zero elements in this SparseTensor
:param i_ndarray zero-based indices for non-zero element
                 i_ndarray's shape should be (shape.size, a_ndarray.size)
                 And the i-th non-zero elements indices is i_ndarray[:, 1],
                 should be zero-based and ascending;
:param shape     shape as a DenseTensor.

>>> import numpy as np
>>> from bigdl.util.common import JTensor
>>> from bigdl.util.common import callBigDlFunc
>>> np.random.seed(123)
>>> data = np.arange(1, 7).astype("float32")
>>> indices = np.arange(1, 7)
>>> shape = np.array([10])
>>> result = JTensor.sparse(data, indices, shape)
>>> expected_storage = np.array([1., 2., 3., 4., 5., 6.])
>>> expected_shape = np.array([10])
>>> expected_indices = np.array([1, 2, 3, 4, 5, 6])
>>> np.testing.assert_allclose(result.storage, expected_storage)
>>> np.testing.assert_allclose(result.shape, expected_shape)
>>> np.testing.assert_allclose(result.indices, expected_indices)
>>> tensor1 = callBigDlFunc("float", "testTensor", result)  # noqa
>>> array_from_tensor = tensor1.to_ndarray()
>>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 0, 0, 0])
>>> (array_from_tensor == expected_ndarray).all()
True


import bigdl.nn.layer as BLayer<EOL>activation = None<EOL>activation_name = activation_name.lower()<EOL>if activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.Tanh()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.Sigmoid()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.HardSigmoid()<EOL><DEDENT>elif activation_name == "<STR_LIT:relu>":<EOL><INDENT>activation = BLayer.ReLU()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.SoftMax()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.SoftPlus(beta=<NUM_LIT:1.0>)<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.SoftSign()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.Identity()<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % activation_name)<EOL><DEDENT>if not activation_id:<EOL><INDENT>activation.set_name(activation_id)<EOL><DEDENT>return activation<EOL>

Convert to a bigdl activation layer
        given the name of the activation as a string


rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer()))<EOL>returnrdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava(<EOL>rdd._jrdd, True)<EOL>

Return a JavaRDD of Object by unpickling

    It will convert each Python object into Java object by Pyrolite, whenever
    the RDD is serialized in batch or not.


gateway = _get_gateway()<EOL>args = [_py2java(gateway, a) for a in args]<EOL>result = func(*args)<EOL>return _java2py(gateway, result)<EOL>


def callBigDlFunc(bigdl_type, name, *args):

gateway = _get_gateway()<EOL>error = Exception("<STR_LIT>" % name)<EOL>for jinvoker in JavaCreator.instance(bigdl_type, gateway).value:<EOL><INDENT>try:<EOL><INDENT>api = getattr(jinvoker, name)<EOL>result = callJavaFunc(api, *args)<EOL><DEDENT>except Exception as e:<EOL><INDENT>error = e<EOL>if "<STR_LIT>" not in str(e):<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return result<EOL><DEDENT><DEDENT>raise error<EOL>


@classmethod<EOL><INDENT>def from_ndarray(cls, a_ndarray, bigdl_type="<STR_LIT:float>"):<DEDENT>

if a_ndarray is None:<EOL><INDENT>return None<EOL><DEDENT>assert isinstance(a_ndarray, np.ndarray),"<STR_LIT>" % type(a_ndarray)<EOL>return cls(a_ndarray,<EOL>a_ndarray.shape if a_ndarray.shape else (a_ndarray.size),<EOL>bigdl_type)<EOL>

Convert a ndarray to a DenseTensor which would be used in Java side.

>>> import numpy as np
>>> from bigdl.util.common import JTensor
>>> from bigdl.util.common import callBigDlFunc
>>> np.random.seed(123)
>>> data = np.random.uniform(0, 1, (2, 3)).astype("float32")
>>> result = JTensor.from_ndarray(data)
>>> expected_storage = np.array([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]])
>>> expected_shape = np.array([2, 3])
>>> np.testing.assert_allclose(result.storage, expected_storage, rtol=1e-6, atol=1e-6)
>>> np.testing.assert_allclose(result.shape, expected_shape)
>>> data_back = result.to_ndarray()
>>> (data == data_back).all()
True
>>> tensor1 = callBigDlFunc("float", "testTensor", JTensor.from_ndarray(data))  # noqa
>>> array_from_tensor = tensor1.to_ndarray()
>>> (array_from_tensor == data).all()
True


assert self.indices is None, "<STR_LIT>"<EOL>return np.array(self.storage, dtype=get_dtype(self.bigdl_type)).reshape(self.shape)<EOL>

Transfer JTensor to ndarray.
As SparseTensor may generate an very big ndarray, so we don't support this function for SparseTensor.
:return: a ndarray


callBigDlFunc(bigdl_type, "<STR_LIT>")<EOL>

Set BigDL log level to INFO.
:param bigdl_type: "double" or "float"


from keras.preprocessing import sequence<EOL>from keras.datasets import imdb<EOL>(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=<NUM_LIT>)<EOL>X_train = sequence.pad_sequences(X_train, maxlen=<NUM_LIT:100>)<EOL>X_test = sequence.pad_sequences(X_test, maxlen=<NUM_LIT:100>)<EOL>return X_train, y_train, X_test, y_test<EOL>

Load IMDB dataset
Transform input data into an RDD of Sample


def compile(self, optimizer, loss, metrics=None):

if isinstance(optimizer, six.string_types):<EOL><INDENT>optimizer = self.__convert_optim_method(optimizer)<EOL><DEDENT>if isinstance(loss, six.string_types):<EOL><INDENT>loss = self.__convert_criterion(loss)<EOL><DEDENT>if all(isinstance(metric, six.string_types) for metric in metrics):<EOL><INDENT>metrics = self.__convert_metrics(metrics)<EOL><DEDENT>callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value,<EOL>optimizer,<EOL>loss,<EOL>metrics)<EOL>

Configures the learning process. Must be called before fit or evaluate.

# Arguments
optimizer: Optimization method to be used. One can alternatively pass in the corresponding
           string representation, such as 'sgd'.
loss: Criterion to be used. One can alternatively pass in the corresponding string
      representation, such as 'mse'.
metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy'].


if is_distributed:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>features = to_sample_rdd(x, np.zeros([x.shape[<NUM_LIT:0>]]))<EOL><DEDENT>elif isinstance(x, RDD):<EOL><INDENT>features = x<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % type(x))<EOL><DEDENT>return self.predict_distributed(features)<EOL><DEDENT>else:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>return self.predict_local(x)<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % type(x))<EOL><DEDENT><DEDENT>

Use a model to do prediction.

# Arguments
x: Input data. A Numpy array or RDD of Sample.
distributed: Boolean. Whether to do prediction in distributed mode or local mode.
             Default is True. In local mode, x must be a Numpy array.


callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value, JTensor.from_ndarray(running_std))<EOL>return self<EOL>

Set the running variance of the BatchNormalization layer.
:param running_std: a Numpy array.


callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value, JTensor.from_ndarray(running_mean))<EOL>return self<EOL>

Set the running mean of the BatchNormalization layer.
:param running_mean: a Numpy array.


return callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value).to_ndarray()<EOL>

Get the running variance of the BatchNormalization layer.


jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", path)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Torch model.

:param path: The path containing the pre-trained model.
:return: A pre-trained model.


result = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>self._to_jtensors(X))<EOL>return np.stack(result)<EOL>

:param X: X can be a ndarray or list of ndarray if the model has multiple inputs.
          The first dimension of X should be batch.
:return: a ndarray as the prediction result.


@staticmethod<EOL><INDENT>def load_caffe_model(defPath, modelPath, bigdl_type="<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", defPath, modelPath)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Caffe model.

:param defPath: The path containing the caffe model definition.
:param modelPath: The path containing the pre-trained caffe model.
:return: A pre-trained model.


@staticmethod<EOL><INDENT>def load_keras(json_path=None, hdf5_path=None, by_name=False):<DEDENT>

import os<EOL>try:<EOL><INDENT>import tensorflow<EOL><DEDENT>except ImportError:<EOL><INDENT>os.environ['<STR_LIT>'] = "<STR_LIT>"<EOL>try:<EOL><INDENT>from theano import ifelse<EOL><DEDENT>except ImportError:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>from bigdl.keras.converter import DefinitionLoader, WeightLoader<EOL>if json_path and not hdf5_path:<EOL><INDENT>return DefinitionLoader.from_json_path(json_path)<EOL><DEDENT>elif json_path and hdf5_path:<EOL><INDENT>return WeightLoader.load_weights_from_json_hdf5(json_path, hdf5_path, by_name=by_name)<EOL><DEDENT>elif hdf5_path and not json_path:<EOL><INDENT>kmodel, bmodel = DefinitionLoader.from_hdf5_path(hdf5_path)<EOL>WeightLoader.load_weights_from_kmodel(bmodel, kmodel)<EOL>return bmodel<EOL><DEDENT>

Load a pre-trained Keras model.

:param json_path: The json path containing the keras model definition.
:param hdf5_path: The HDF5 path containing the pre-trained keras model weights with or without the model architecture.
:return: A bigdl model.


@staticmethod<EOL><INDENT>def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", model, defPath, modelPath, match_all)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Caffe model.

:param model: A bigdl model definition \which equivalent to the pre-trained caffe model.
:param defPath: The path containing the caffe model definition.
:param modelPath: The path containing the pre-trained caffe model.
:return: A pre-trained model.


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, names)<EOL>return self<EOL>

unfreeze module, if names is not None, unfreeze layers that match given names
:param names: an array of layer names
:return:


model = Model([], [], jvalue=jvalue)<EOL>model.value = jvalue<EOL>return model<EOL>

Create a Python Model base on the given java value
:param jvalue: Java object create by Py4j
:return: A Python Model


def to_jtensor(i):<EOL><INDENT>if isinstance(i, np.ndarray):<EOL><INDENT>return JTensor.from_ndarray(i)<EOL><DEDENT>elif isinstance(i, JTensor):<EOL><INDENT>return i<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % type(i))<EOL><DEDENT><DEDENT>if type(input) is list:<EOL><INDENT>if len(input) == <NUM_LIT:0>:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>return list(map(lambda i: to_jtensor(i), input)), True<EOL><DEDENT>else:<EOL><INDENT>return [to_jtensor(input)], False<EOL><DEDENT>

:param input: ndarray or list of ndarray or JTensor or list of JTensor.
:return: (list of JTensor, isTable)


Load a pre-trained Bigdl model.

:param path: The path containing the pre-trained model.
:return: A pre-trained model.


if len(args) == <NUM_LIT:0>:<EOL><INDENT>callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value)<EOL>return self<EOL><DEDENT>elif len(args) == <NUM_LIT:3>:<EOL><INDENT>dataset, batch_size, val_methods = args<EOL>if (isinstance(dataset, ImageFrame)):<EOL><INDENT>return callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>dataset, batch_size, val_methods)<EOL><DEDENT>else:<EOL><INDENT>return callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>dataset, batch_size, val_methods)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>

No argument passed in:
Evaluate the model to set train = false, useful when doing test/forward
:return: layer itself

Three arguments passed in:
A method to benchmark the model quality.

:param dataset: the input data
:param batch_size: batch size
:param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss.
:return: a list of the metrics result


callJavaFunc(self.value.setName, name)<EOL>return self<EOL>

Give this model a name. There would be a generated name
consist of class name and UUID if user doesn't set it.


if isinstance(features, RDD):<EOL><INDENT>return self.predict_class_distributed(features)<EOL><DEDENT>else:<EOL><INDENT>return self.predict_class_local(features)<EOL><DEDENT>

Model inference base on the given data which returning label
:param features: it can be a ndarray or list of ndarray for locally inference
                 or RDD[Sample] for running in distributed fashion
:return: ndarray or RDD[Sample] depend on the the type of features.


@staticmethod<EOL><INDENT>def loadModel(modelPath, weightPath =None, bigdl_type="<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", modelPath, weightPath)<EOL>return Layer.of(jmodel)<EOL>


if is_training:<EOL><INDENT>callJavaFunc(self.value.training)<EOL><DEDENT>else:<EOL><INDENT>callJavaFunc(self.value.evaluate)<EOL><DEDENT>return self<EOL>

Set this layer in the training mode or in predition mode if is_training=False


self.value.bRegularizer = bRegularizer.value<EOL>

set bias regularizer
:param wRegularizer: bias regularizer
:return:


def __init__(self, max, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, max)<EOL>

Create a MaxIteration trigger.

:param max: max


JavaValue.__init__(self, None, bigdl_type)<EOL>


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>summary)<EOL>return self<EOL>

Set validation summary. A ValidationSummary object contains information
necessary for the optimizer to know how often the logs are recorded,
where to store the logs and how to retrieve them, etc. For details,
refer to the docs of ValidationSummary.

:param summary: a ValidationSummary object


def set_traindata(self, training_rdd, batch_size):

callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>training_rdd, batch_size)<EOL>

Set new training dataset, for optimizer reuse

:param training_rdd: the training dataset
:param batch_size: training batch size
:return:


def __init__(self, log_dir, app_name, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, log_dir, app_name)<EOL>

Create a TrainSummary. Logs will be saved to log_dir/app_name/train.

:param log_dir: the root dir to store the logs
:param app_name: the application name


def __init__(self, min, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, min)<EOL>

Create a MinLoss trigger.

:param min: min loss


self.value.setModel(model.value)<EOL>


JavaValue.__init__(self, None, "<STR_LIT:float>", first, list(other))<EOL>

Create a And trigger.

:param first: first Trigger
:param other: other Trigger


@classmethod<EOL><INDENT>def read_parquet(cls, path, sc, bigdl_type="<STR_LIT:float>"):<DEDENT>

return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "<STR_LIT>", path, sc))<EOL>

Read parquet file as DistributedImageFrame


return callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>


def get_image(self, float_key="<STR_LIT>", to_chw=True):

return self.image_frame.get_image(float_key, to_chw)<EOL>


return self.image_frame.get_predict(key)<EOL>


def set_label(self, label, bigdl_type="<STR_LIT:float>"):

return callBigDlFunc(bigdl_type,<EOL>"<STR_LIT>", label, self.value)<EOL>


@classmethod<EOL><INDENT>def read(cls, path, sc=None, min_partitions=<NUM_LIT:1>, bigdl_type="<STR_LIT:float>"):<DEDENT>

return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "<STR_LIT>", path, sc, min_partitions))<EOL>

Read images as Image Frame
if sc is defined, Read image as DistributedImageFrame from local file system or HDFS
if sc is null, Read image as LocalImageFrame from local file system
:param path path to read images
if sc is defined, path can be local or HDFS. Wildcard character are supported.
if sc is null, path is local directory/image file/image file with wildcard character
:param sc SparkContext
:param min_partitions A suggestion value of the minimal splitting number for input data.
:return ImageFrame


label = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>return label.to_ndarray()<EOL>


sparkConf = create_spark_conf().setMaster("<STR_LIT>").setAppName("<STR_LIT>")<EOL>self.sc = get_spark_context(sparkConf)<EOL>init_engine()<EOL>self.resource_path = os.path.join(os.path.split(__file__)[<NUM_LIT:0>], "<STR_LIT>")<EOL>

setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.


self.sc.stop()<EOL>

teardown any state that was previously setup with a setup_method
call.


sparkConf = create_spark_conf().setMaster("<STR_LIT>").setAppName("<STR_LIT>")<EOL>self.sc = get_spark_context(sparkConf)<EOL>self.sqlContext = SQLContext(self.sc)<EOL>init_engine()<EOL>

setup any state tied to the execution of the given method in a
        class.  setup_method is invoked for every test method of a class.


sparkConf = create_spark_conf().setMaster("<STR_LIT>").setAppName("<STR_LIT>")<EOL>self.sc = get_spark_context(sparkConf)<EOL>init_engine()<EOL>resource_path = os.path.join(os.path.split(__file__)[<NUM_LIT:0>], "<STR_LIT>")<EOL>self.image_path = os.path.join(resource_path, "<STR_LIT>")<EOL>


self.sc.stop()<EOL>

teardown any state that was previously setup with a setup_method
        call.


line = logical_line<EOL>if line.startswith('<STR_LIT>'):<EOL><INDENT>found = line.find('<STR_LIT:U+002C>')<EOL>if -<NUM_LIT:1> < found and '<STR_LIT:;>' not in line[:found]:<EOL><INDENT>yield found, "<STR_LIT>"<EOL><DEDENT><DEDENT>

r"""Imports should usually be on separate lines.

    Okay: import os\nimport sys
    E401: import sys, os

    Okay: from subprocess import Popen, PIPE
    Okay: from myclas import MyClass
    Okay: from foo.bar.yourclass import YourClass
    Okay: import myclass
    Okay: import foo.bar.yourclass


def run_check(self, check, argument_names):

arguments = []<EOL>for name in argument_names:<EOL><INDENT>arguments.append(getattr(self, name))<EOL><DEDENT>return check(*arguments)<EOL>


if paths is None:<EOL><INDENT>paths = self.paths<EOL><DEDENT>report = self.options.report<EOL>runner = self.runner<EOL>report.start()<EOL>try:<EOL><INDENT>for path in paths:<EOL><INDENT>if os.path.isdir(path):<EOL><INDENT>self.input_dir(path)<EOL><DEDENT>elif not self.excluded(path):<EOL><INDENT>runner(path)<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>report.stop()<EOL>return report<EOL>


return self.file_errors<EOL>

Return the count of errors and warnings for this file.


def _add_check(check, kind, codes, args):<EOL><INDENT>if check in _checks[kind]:<EOL><INDENT>_checks[kind][check][<NUM_LIT:0>].extend(codes or [])<EOL><DEDENT>else:<EOL><INDENT>_checks[kind][check] = (codes or ['<STR_LIT>'], args)<EOL><DEDENT><DEDENT>if inspect.isfunction(check):<EOL><INDENT>args = _get_parameters(check)<EOL>if args and args[<NUM_LIT:0>] in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if codes is None:<EOL><INDENT>codes = ERRORCODE_REGEX.findall(check.__doc__ or '<STR_LIT>')<EOL><DEDENT>_add_check(check, args[<NUM_LIT:0>], codes, args)<EOL><DEDENT><DEDENT>elif inspect.isclass(check):<EOL><INDENT>if _get_parameters(check.__init__)[:<NUM_LIT:2>] == ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>_add_check(check, '<STR_LIT>', codes, None)<EOL><DEDENT><DEDENT>


if '<STR_LIT:\t>' not in line:<EOL><INDENT>return len(line) - len(line.lstrip())<EOL><DEDENT>result = <NUM_LIT:0><EOL>for char in line:<EOL><INDENT>if char == '<STR_LIT:\t>':<EOL><INDENT>result = result // <NUM_LIT:8> * <NUM_LIT:8> + <NUM_LIT:8><EOL><DEDENT>elif char == '<STR_LIT:U+0020>':<EOL><INDENT>result += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return result<EOL>

r"""Return the amount of indentation.

    Tabs are expanded to the next multiple of 8.

    >>> expand_indent('    ')
    4
    >>> expand_indent('\t')
    8
    >>> expand_indent('       \t')
    8
    >>> expand_indent('        \t')
    16


for match in OPERATOR_REGEX.finditer(logical_line):<EOL><INDENT>before, after = match.groups()<EOL>if '<STR_LIT:\t>' in before:<EOL><INDENT>yield match.start(<NUM_LIT:1>), "<STR_LIT>"<EOL><DEDENT>elif len(before) > <NUM_LIT:1>:<EOL><INDENT>yield match.start(<NUM_LIT:1>), "<STR_LIT>"<EOL><DEDENT>if '<STR_LIT:\t>' in after:<EOL><INDENT>yield match.start(<NUM_LIT:2>), "<STR_LIT>"<EOL><DEDENT>elif len(after) > <NUM_LIT:1>:<EOL><INDENT>yield match.start(<NUM_LIT:2>), "<STR_LIT>"<EOL><DEDENT><DEDENT>

r"""Avoid extraneous whitespace around an operator.

    Okay: a = 12 + 3
    E221: a = 4  + 5
    E222: a = 4 +  5
    E223: a = 4\t+ 5
    E224: a = 4 +\t5


indent = INDENT_REGEX.match(physical_line).group(<NUM_LIT:1>)<EOL>for offset, char in enumerate(indent):<EOL><INDENT>if char != indent_char:<EOL><INDENT>return offset, "<STR_LIT>"<EOL><DEDENT><DEDENT>

r"""Never mix tabs and spaces.

    The most popular way of indenting Python is with spaces only.  The
    second-most popular way is with tabs only.  Code indented with a mixture
    of tabs and spaces should be converted to using spaces exclusively.  When
    invoking the Python command line interpreter with the -t option, it issues
    warnings about code that illegally mixes tabs and spaces.  When using -tt
    these warnings become errors.  These options are highly recommended!

    Okay: if a == 0:\n        a = 1\n        b = 1
    E101: if a == 0:\n        a = 1\n\tb = 1


mod = inspect.getmodule(register_check)<EOL>for (name, function) in inspect.getmembers(mod, inspect.isfunction):<EOL><INDENT>register_check(function)<EOL><DEDENT>

Register all globally visible functions.

    The first argument name is either 'physical_line' or 'logical_line'.


def error(self, line_number, offset, text, check):

code = text[:<NUM_LIT:4>]<EOL>if self._ignore_code(code):<EOL><INDENT>return<EOL><DEDENT>if code in self.counters:<EOL><INDENT>self.counters[code] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self.counters[code] = <NUM_LIT:1><EOL>self.messages[code] = text[<NUM_LIT:5>:]<EOL><DEDENT>if code in self.expected:<EOL><INDENT>return<EOL><DEDENT>if self.print_filename and not self.file_errors:<EOL><INDENT>print(self.filename)<EOL><DEDENT>self.file_errors += <NUM_LIT:1><EOL>self.total_errors += <NUM_LIT:1><EOL>return code<EOL>


prev_type, prev_text, __, prev_end, __ = tokens[<NUM_LIT:0>]<EOL>for index in range(<NUM_LIT:1>, len(tokens)):<EOL><INDENT>token_type, text, start, end, __ = tokens[index]<EOL>if (token_type == tokenize.OP and<EOL>text in '<STR_LIT>' and<EOL>start != prev_end and<EOL>(prev_type == tokenize.NAME or prev_text in '<STR_LIT>') and<EOL>(index < <NUM_LIT:2> or tokens[index - <NUM_LIT:2>][<NUM_LIT:1>] != '<STR_LIT:class>') and<EOL>not keyword.iskeyword(prev_text)):<EOL><INDENT>yield prev_end, "<STR_LIT>" % text<EOL><DEDENT>prev_type = token_type<EOL>prev_text = text<EOL>prev_end = end<EOL><DEDENT>

r"""Avoid extraneous whitespace.

    Avoid extraneous whitespace in the following situations:
    - before the open parenthesis that starts the argument list of a
      function call.
    - before the open parenthesis that starts an indexing or slicing.

    Okay: spam(1)
    E211: spam (1)

    Okay: dict['key'] = list[index]
    E211: dict ['key'] = list[index]
    E211: dict['key'] = list [index]


pos = logical_line.find('<STR_LIT>')<EOL>if pos > -<NUM_LIT:1>:<EOL><INDENT>yield pos, "<STR_LIT>"<EOL><DEDENT>

r"""New code should always use != instead of <>.

    The older syntax is removed in Python 3.

    Okay: if a != 'no':
    W603: if a <> 'no':


if self.line_number >= self.total_lines:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>line = self.lines[self.line_number]<EOL>self.line_number += <NUM_LIT:1><EOL>if self.indent_char is None and line[:<NUM_LIT:1>] in WHITESPACE:<EOL><INDENT>self.indent_char = line[<NUM_LIT:0>]<EOL><DEDENT>return line<EOL>

Get the next line from the input buffer.


self.counters['<STR_LIT>'] += <NUM_LIT:1><EOL>


dirname = dirname.rstrip('<STR_LIT:/>')<EOL>if self.excluded(dirname):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>counters = self.options.report.counters<EOL>verbose = self.options.verbose<EOL>filepatterns = self.options.filename<EOL>runner = self.runner<EOL>for root, dirs, files in os.walk(dirname):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>' + root)<EOL><DEDENT>counters['<STR_LIT>'] += <NUM_LIT:1><EOL>for subdir in sorted(dirs):<EOL><INDENT>if self.excluded(subdir, root):<EOL><INDENT>dirs.remove(subdir)<EOL><DEDENT><DEDENT>for filename in sorted(files):<EOL><INDENT>if ((filename_match(filename, filepatterns) and<EOL>not self.excluded(filename, root))):<EOL><INDENT>runner(os.path.join(root, filename))<EOL><DEDENT><DEDENT><DEDENT>

Check all files in this directory and all subdirectories.


print('<STR_LIT>' % (self.elapsed, '<STR_LIT>'))<EOL>if self.elapsed:<EOL><INDENT>for key in self._benchmark_keys:<EOL><INDENT>print('<STR_LIT>' %<EOL>(self.counters[key] / self.elapsed, key,<EOL>self.counters[key]))<EOL><DEDENT><DEDENT>


for line in self.get_statistics(prefix):<EOL><INDENT>print(line)<EOL><DEDENT>

Print overall statistics (number of errors and warnings).


line = logical_line<EOL>last_char = len(line) - <NUM_LIT:1><EOL>found = line.find('<STR_LIT::>')<EOL>while -<NUM_LIT:1> < found < last_char:<EOL><INDENT>before = line[:found]<EOL>if ((before.count('<STR_LIT:{>') <= before.count('<STR_LIT:}>') and   <EOL>before.count('<STR_LIT:[>') <= before.count('<STR_LIT:]>') and   <EOL>before.count('<STR_LIT:(>') <= before.count('<STR_LIT:)>'))):    <EOL><INDENT>lambda_kw = LAMBDA_REGEX.search(before)<EOL>if lambda_kw:<EOL><INDENT>before = line[:lambda_kw.start()].rstrip()<EOL>if before[-<NUM_LIT:1>:] == '<STR_LIT:=>' and isidentifier(before[:-<NUM_LIT:1>].strip()):<EOL><INDENT>yield <NUM_LIT:0>, ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>break<EOL><DEDENT>if before.startswith('<STR_LIT>'):<EOL><INDENT>yield <NUM_LIT:0>, "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>yield found, "<STR_LIT>"<EOL><DEDENT><DEDENT>found = line.find('<STR_LIT::>', found + <NUM_LIT:1>)<EOL><DEDENT>found = line.find('<STR_LIT:;>')<EOL>while -<NUM_LIT:1> < found:<EOL><INDENT>if found < last_char:<EOL><INDENT>yield found, "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>yield found, "<STR_LIT>"<EOL><DEDENT>found = line.find('<STR_LIT:;>', found + <NUM_LIT:1>)<EOL><DEDENT>

r"""Compound statements (on the same line) are generally discouraged.

    While sometimes it's okay to put an if/for/while with a small body
    on the same line, never do this for multi-clause statements.
    Also avoid folding such long lines!

    Always use a def statement instead of an assignment statement that
    binds a lambda expression directly to a name.

    Okay: if foo == 'blah':\n    do_blah_thing()
    Okay: do_one()
    Okay: do_two()
    Okay: do_three()

    E701: if foo == 'blah': do_blah_thing()
    E701: for x in lst: total += x
    E701: while t < 10: t = delay()
    E701: if foo == 'blah': do_blah_thing()
    E701: else: do_non_blah_thing()
    E701: try: something()
    E701: finally: cleanup()
    E701: if foo == 'blah': one(); two(); three()
    E702: do_one(); do_two(); do_three()
    E703: do_four();  # useless semicolon
    E704: def f(x): return 2*x
    E731: f = lambda x: 2*x


self.options.report = (reporter or self.options.reporter)(self.options)<EOL>return self.options.report<EOL>


def is_binary_operator(token_type, text):<EOL><INDENT>return ((token_type == tokenize.OP or text in ['<STR_LIT>', '<STR_LIT>']) and<EOL>text not in "<STR_LIT>")<EOL><DEDENT>line_break = False<EOL>unary_context = True<EOL>for token_type, text, start, end, line in tokens:<EOL><INDENT>if token_type == tokenize.COMMENT:<EOL><INDENT>continue<EOL><DEDENT>if ('<STR_LIT:\n>' in text or '<STR_LIT:\r>' in text) and token_type != tokenize.STRING:<EOL><INDENT>line_break = True<EOL><DEDENT>else:<EOL><INDENT>if (is_binary_operator(token_type, text) and line_break and<EOL>not unary_context):<EOL><INDENT>yield start, "<STR_LIT>"<EOL><DEDENT>unary_context = text in '<STR_LIT>'<EOL>line_break = False<EOL><DEDENT><DEDENT>

r"""
    Avoid breaks before binary operators.

    The preferred place to break around a binary operator is after the
    operator, not before it.

    W503: (width == 0\n + height == 0)
    W503: (width == 0\n and height == 0)

    Okay: (width == 0 +\n height == 0)
    Okay: foo(\n    -x)
    Okay: foo(x\n    [])
    Okay: x = '''\n''' + ''
    Okay: foo(x,\n    -y)
    Okay: foo(x,  # comment\n    -y)


if not value:<EOL><INDENT>return []<EOL><DEDENT>if isinstance(value, list):<EOL><INDENT>return value<EOL><DEDENT>paths = []<EOL>for path in value.split('<STR_LIT:U+002C>'):<EOL><INDENT>path = path.strip()<EOL>if '<STR_LIT:/>' in path:<EOL><INDENT>path = os.path.abspath(os.path.join(parent, path))<EOL><DEDENT>paths.append(path.rstrip('<STR_LIT:/>'))<EOL><DEDENT>return paths<EOL>

Parse a comma-separated list of paths.

    Return a list of absolute paths.


parens = <NUM_LIT:0><EOL>no_space = False<EOL>prev_end = None<EOL>annotated_func_arg = False<EOL>in_def = logical_line.startswith('<STR_LIT>')<EOL>message = "<STR_LIT>"<EOL>for token_type, text, start, end, line in tokens:<EOL><INDENT>if token_type == tokenize.NL:<EOL><INDENT>continue<EOL><DEDENT>if no_space:<EOL><INDENT>no_space = False<EOL>if start != prev_end:<EOL><INDENT>yield (prev_end, message)<EOL><DEDENT><DEDENT>if token_type == tokenize.OP:<EOL><INDENT>if text == '<STR_LIT:(>':<EOL><INDENT>parens += <NUM_LIT:1><EOL><DEDENT>elif text == '<STR_LIT:)>':<EOL><INDENT>parens -= <NUM_LIT:1><EOL><DEDENT>elif in_def and text == '<STR_LIT::>' and parens == <NUM_LIT:1>:<EOL><INDENT>annotated_func_arg = True<EOL><DEDENT>elif parens and text == '<STR_LIT:U+002C>' and parens == <NUM_LIT:1>:<EOL><INDENT>annotated_func_arg = False<EOL><DEDENT>elif parens and text == '<STR_LIT:=>' and not annotated_func_arg:<EOL><INDENT>no_space = True<EOL>if start != prev_end:<EOL><INDENT>yield (prev_end, message)<EOL><DEDENT><DEDENT>if not parens:<EOL><INDENT>annotated_func_arg = False<EOL><DEDENT><DEDENT>prev_end = end<EOL><DEDENT>

r"""Don't use spaces around the '=' sign in function arguments.

    Don't use spaces around the '=' sign when used to indicate a
    keyword argument or a default parameter value.

    Okay: def complex(real, imag=0.0):
    Okay: return magic(r=real, i=imag)
    Okay: boolean(a == b)
    Okay: boolean(a != b)
    Okay: boolean(a <= b)
    Okay: boolean(a >= b)
    Okay: def foo(arg: int = 42):

    E251: def complex(real, imag = 0.0):
    E251: return magic(r = real, i = imag)


def filename_match(filename, patterns, default=True):

if not patterns:<EOL><INDENT>return default<EOL><DEDENT>return any(fnmatch(filename, pattern) for pattern in patterns)<EOL>

Check if patterns contains a pattern that matches filename.

    If patterns is unspecified, this always returns True.


<EOL>start = text.index(text[-<NUM_LIT:1>]) + <NUM_LIT:1><EOL>end = len(text) - <NUM_LIT:1><EOL>if text[-<NUM_LIT:3>:] in ('<STR_LIT>', "<STR_LIT>"):<EOL><INDENT>start += <NUM_LIT:2><EOL>end -= <NUM_LIT:2><EOL><DEDENT>return text[:start] + '<STR_LIT:x>' * (end - start) + text[end:]<EOL>

Replace contents with 'xxx' to prevent syntax matching.

    >>> mute_string('"abc"')
    '"xxx"'
    >>> mute_string("'''abc'''")
    "'''xxx'''"
    >>> mute_string("r'abc'")
    "r'xxx'"


if self._io_error:<EOL><INDENT>self.report_error(<NUM_LIT:1>, <NUM_LIT:0>, '<STR_LIT>' % self._io_error, readlines)<EOL><DEDENT>tokengen = tokenize.generate_tokens(self.readline)<EOL>try:<EOL><INDENT>for token in tokengen:<EOL><INDENT>if token[<NUM_LIT:2>][<NUM_LIT:0>] > self.total_lines:<EOL><INDENT>return<EOL><DEDENT>self.maybe_check_physical(token)<EOL>yield token<EOL><DEDENT><DEDENT>except (SyntaxError, tokenize.TokenError):<EOL><INDENT>self.report_invalid_syntax()<EOL><DEDENT>

Tokenize the file, run physical line checks and yield tokens.


def init_file(self, filename, lines, expected, line_offset):

self.filename = filename<EOL>self.lines = lines<EOL>self.expected = expected or ()<EOL>self.line_offset = line_offset<EOL>self.file_errors = <NUM_LIT:0><EOL>self.counters['<STR_LIT>'] += <NUM_LIT:1><EOL>self.counters['<STR_LIT>'] += len(lines)<EOL>


match = RAISE_COMMA_REGEX.match(logical_line)<EOL>if match and not RERAISE_COMMA_REGEX.match(logical_line):<EOL><INDENT>yield match.end() - <NUM_LIT:1>, "<STR_LIT>"<EOL><DEDENT>

r"""When raising an exception, use "raise ValueError('message')".

    The older form is removed in Python 3.

    Okay: raise DummyError("Message")
    W602: raise DummyError, "Message"


if zone.upper() == '<STR_LIT>':<EOL><INDENT>return utc<EOL><DEDENT>try:<EOL><INDENT>zone = ascii(zone)<EOL><DEDENT>except UnicodeEncodeError:<EOL><INDENT>raise UnknownTimeZoneError(zone)<EOL><DEDENT>zone = _unmunge_zone(zone)<EOL>if zone not in _tzinfo_cache:<EOL><INDENT>if zone in all_timezones_set:<EOL><INDENT>fp = open_resource(zone)<EOL>try:<EOL>

r''' Return a datetime.tzinfo implementation for the given timezone

    >>> from datetime import datetime, timedelta
    >>> utc = timezone('UTC')
    >>> eastern = timezone('US/Eastern')
    >>> eastern.zone
    'US/Eastern'
    >>> timezone(unicode('US/Eastern')) is eastern
    True
    >>> utc_dt = datetime(2002, 10, 27, 6, 0, 0, tzinfo=utc)
    >>> loc_dt = utc_dt.astimezone(eastern)
    >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)'
    >>> loc_dt.strftime(fmt)
    '2002-10-27 01:00:00 EST (-0500)'
    >>> (loc_dt - timedelta(minutes=10)).strftime(fmt)
    '2002-10-27 00:50:00 EST (-0500)'
    >>> eastern.normalize(loc_dt - timedelta(minutes=10)).strftime(fmt)
    '2002-10-27 01:50:00 EDT (-0400)'
    >>> (loc_dt + timedelta(minutes=10)).strftime(fmt)
    '2002-10-27 01:10:00 EST (-0500)'

    Raises UnknownTimeZoneError if passed an unknown zone.

    >>> try:
    ...     timezone('Asia/Shangri-La')
    ... except UnknownTimeZoneError:
    ...     print('Unknown')
    Unknown

    >>> try:
    ...     timezone(unicode('\N{TRADE MARK SIGN}'))
    ... except UnknownTimeZoneError:
    ...     print('Unknown')
    Unknown


return td.seconds + td.days * <NUM_LIT> * <NUM_LIT> * <NUM_LIT><EOL>


return self._utcoffset<EOL>

See datetime.tzinfo.utcoffset

        is_dst is ignored for StaticTzInfo, and exists only to
        retain compatibility with DstTzInfo.


def plexp_inv(P, xmin, alpha, guess=<NUM_LIT:1.>):

def equation(x,prob):<EOL><INDENT>return plexp_cdf(x, xmin, alpha)-prob<EOL><DEDENT>def solver(y, x0=guess):<EOL><INDENT>return scipy.optimize.fsolve(equation, guess, args=(y,))<EOL><DEDENT>f = np.vectorize(solver)<EOL>return f(P)<EOL>

Inverse CDF for a piecewise PDF as defined in eqn. 3.10
of Clauset et al.

(previous version was incorrect and lead to weird discontinuities in the
distribution function)


Plot alpha versus the ks value for derived alpha.  This plot can be used
as a diagnostic of whether you have derived the 'best' fit: if there are
multiple local minima, your data set may be well suited to a broken
powerlaw or a different function.


def plfit(self, nosmall=True, finite=False, quiet=False, silent=False,<EOL>usefortran=False, usecy=False, xmin=None, verbose=False,<EOL>discrete=None, discrete_approx=True, discrete_n_alpha=<NUM_LIT:1000>,<EOL>skip_consistency_check=False):

x = self.data<EOL>if any(x < <NUM_LIT:0>):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>z = np.sort(x)<EOL>xmins,argxmins = np.unique(z,return_index=True)<EOL>self._nunique = len(xmins)<EOL>if self._nunique == len(x) and discrete is None:<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>discrete = False<EOL><DEDENT>elif self._nunique < len(x) and discrete is None:<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>discrete = True<EOL><DEDENT>t = time.time()<EOL>if xmin is None:<EOL><INDENT>if discrete:<EOL><INDENT>self.discrete_best_alpha(approximate=discrete_approx,<EOL>n_alpha=discrete_n_alpha,<EOL>verbose=verbose,<EOL>finite=finite)<EOL>return self._xmin,self._alpha<EOL><DEDENT>elif usefortran and fortranOK:<EOL><INDENT>kstest_values,alpha_values = fplfit.plfit(z, <NUM_LIT:0>)<EOL>if not quiet:<EOL><INDENT>print(("<STR_LIT>" % (time.time()-t)))<EOL><DEDENT><DEDENT>elif usecy and cyOK:<EOL><INDENT>kstest_values,alpha_values = cplfit.plfit_loop(z,<EOL>nosmall=False,<EOL>zunique=xmins,<EOL>argunique=argxmins)<EOL>if not quiet:<EOL><INDENT>print(("<STR_LIT>" % (time.time()-t)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>f_alpha = alpha_gen(z)<EOL>f_kstest = kstest_gen(z)<EOL>alpha_values = np.asarray(list(map(f_alpha,xmins)),<EOL>dtype='<STR_LIT:float>')<EOL>kstest_values = np.asarray(list(map(f_kstest,xmins)),<EOL>dtype='<STR_LIT:float>')<EOL>if not quiet:<EOL><INDENT>print(("<STR_LIT>" % (time.time()-t)))<EOL><DEDENT><DEDENT>if not quiet:<EOL><INDENT>if usefortran and not fortranOK:<EOL><INDENT>raise ImportError("<STR_LIT>")<EOL><DEDENT>if usecy and not cyOK:<EOL><INDENT>raise ImportError("<STR_LIT>")<EOL><DEDENT><DEDENT>sigma = (alpha_values-<NUM_LIT:1>)/np.sqrt(len(z)-argxmins)<EOL>if nosmall:<EOL><INDENT>goodvals = sigma<<NUM_LIT:0.1><EOL>nmax = argmin(goodvals)<EOL>if nmax <= <NUM_LIT:0>:<EOL><INDENT>nmax = len(xmins) - <NUM_LIT:1><EOL>if not silent:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>nmax = len(xmins)-<NUM_LIT:1><EOL><DEDENT>best_ks_index = argmin(kstest_values[:nmax])<EOL>xmin = xmins[best_ks_index]<EOL>self._alpha_values = alpha_values<EOL>self._xmin_kstest = kstest_values<EOL>if scipyOK:<EOL><INDENT>self._ks_prob_all = np.array([scipy.stats.ksone.sf(D_stat,<EOL>len(kstest_values)-ii)<EOL>for ii,D_stat in<EOL>enumerate(kstest_values)])<EOL><DEDENT>self._sigma = sigma<EOL>n = np.count_nonzero(z>=xmin)<EOL>alpha = <NUM_LIT:1.> + float(n)/sum(log(z[z>=xmin]/xmin))<EOL>try:<EOL><INDENT>if not skip_consistency_check:<EOL><INDENT>np.testing.assert_almost_equal(alpha,<EOL>alpha_values[best_ks_index],<EOL>decimal=<NUM_LIT:4>)<EOL><DEDENT><DEDENT>except AssertionError:<EOL><INDENT>raise AssertionError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>.format(alpha,<EOL>alpha_values[best_ks_index]))<EOL><DEDENT><DEDENT>z = z[z>=xmin]<EOL>n = len(z)<EOL>alpha = <NUM_LIT:1.> + float(n) / sum(log(z/xmin))<EOL>if finite:<EOL><INDENT>alpha = alpha*(n-<NUM_LIT:1.>)/n+<NUM_LIT:1.>/n<EOL><DEDENT>if n < <NUM_LIT:50> and not finite and not silent:<EOL><INDENT>print(('<STR_LIT>' % n))<EOL><DEDENT>ks = max(abs( np.arange(n)/float(n) - (<NUM_LIT:1>-(xmin/z)**(alpha-<NUM_LIT:1>)) ))<EOL>L = n*log((alpha-<NUM_LIT:1>)/xmin) - alpha*sum(log(z/xmin))<EOL>self._likelihood = L<EOL>self._xmin = xmin<EOL>self._xmins = xmins<EOL>self._alpha= alpha<EOL>self._alphaerr = (alpha-<NUM_LIT:1>)/np.sqrt(n)<EOL>self._ks = ks<EOL>if scipyOK:<EOL><INDENT>self._ks_prob = scipy.stats.ksone.sf(ks, n)<EOL><DEDENT>self._ngtx = n<EOL>if n == <NUM_LIT:1>:<EOL><INDENT>if not silent:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>self._alpha = alpha = <NUM_LIT:0><EOL>self._alphaerr = <NUM_LIT:0><EOL>self._likelihood = L = <NUM_LIT:0><EOL>self._ks = <NUM_LIT:0><EOL>self._ks_prob = <NUM_LIT:0><EOL>self._xmin = xmin<EOL>return xmin,<NUM_LIT:0><EOL><DEDENT>if np.isnan(L) or np.isnan(xmin) or np.isnan(alpha):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not quiet:<EOL><INDENT>if verbose: print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>" % xmin, end='<STR_LIT:U+0020>')<EOL>if verbose: print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>" % n, end='<STR_LIT:U+0020>')<EOL>if verbose: print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>" % (alpha,self._alphaerr), end='<STR_LIT:U+0020>')<EOL>if verbose: print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>" % L, end='<STR_LIT:U+0020>')<EOL>if verbose: print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>" % (ks), end='<STR_LIT:U+0020>')<EOL>if scipyOK:<EOL><INDENT>if verbose: print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>" % (self._ks_prob))<EOL><DEDENT>else:<EOL><INDENT>print()<EOL><DEDENT><DEDENT>return xmin,alpha<EOL>

A Python implementation of the Matlab code
http://www.santafe.edu/~aaronc/powerlaws/plfit.m
from http://www.santafe.edu/~aaronc/powerlaws/

See A. Clauset, C.R. Shalizi, and M.E.J. Newman, "Power-law distributions
in empirical data" SIAM Review, 51, 661-703 (2009). (arXiv:0706.1062)
http://arxiv.org/abs/0706.1062

There are 3 implementations of xmin estimation.  The fortran version is
fastest, the C (cython) version is ~10% slower, and the python version
is ~3x slower than the fortran version.  Also, the cython code suffers
~2% numerical error relative to the fortran and python for unknown
reasons.

There is also a discrete version implemented in python - it is
different from the continous version!

Parameters
----------
discrete : bool or None
    If *discrete* is None, the code will try to determine whether the
    data set is discrete or continous based on the uniqueness of the
    data; if your data set is continuous but you have any non-unique
    data points (e.g., flagged "bad" data), the "automatic"
    determination will fail.  If *discrete* is True or False, the
    discrete or continuous fitter will be used, respectively.
xmin : float or int
    If you specify xmin, the fitter will only determine alpha assuming
    the given xmin; the rest of the code (and most of the complexity)
    is determining an estimate for xmin and alpha.
nosmall : bool
    When on, the code rejects low s/n points.  WARNING: This option,
    which is on by default, may result in different answers than the
    original Matlab code and the "powerlaw" python package
finite : bool
    There is a 'finite-size bias' to the estimator.  The "alpha" the
    code measures is "alpha-hat" s.t. ᾶ = (nα-1)/(n-1), or α = (1 + ᾶ
    (n-1)) / n
quiet : bool
    If False, delivers messages about what fitter is used and the fit
    results
verbose : bool
    Deliver descriptive messages about the fit parameters (only if
    `quiet==False`)
silent : bool
    If True, will print NO messages
skip_consistency_check : bool
    The code will normally perform a consistency check to make sure the
    alpha value computed by the fitter matches the alpha value computed
    directly in python.  It is possible for numerical differences to
    creep in, usually at the 10^-6 or less level.  If you see an
    exception reporting this type of error, skipping the check can be
    the appropriate next step.

Returns
-------
(xmin, alpha)
The best-fit xmin and alpha values


x = np.array(x) <EOL>if (x<<NUM_LIT:0>).sum() > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % ((x<<NUM_LIT:0>).sum()))<EOL>x = x[x><NUM_LIT:0>]<EOL><DEDENT>self.data = x<EOL>self.plfit(**kwargs)<EOL>

Initializes and fits the power law.  Can pass "quiet" to turn off
output (except for warnings; "silent" turns off warnings)


if not hasattr(self,'<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>x=np.sort(self.data)<EOL>n=len(x)<EOL>xcdf = np.arange(n,<NUM_LIT:0>,-<NUM_LIT:1>,dtype='<STR_LIT:float>')/float(n)<EOL>lcdf = self.lognormal_dist.sf(x)<EOL>D_location = argmax(xcdf-lcdf)<EOL>pylab.vlines(x[D_location],xcdf[D_location],lcdf[D_location],color='<STR_LIT:m>',linewidth=<NUM_LIT:2>)<EOL>pylab.plot(x, lcdf,'<STR_LIT:U+002C>',**kwargs)<EOL>


x = (<NUM_LIT:1>-P)**(<NUM_LIT:1>/(<NUM_LIT:1>-a)) * xm<EOL>return x<EOL>


return (alpha-<NUM_LIT:1.>) / n**<NUM_LIT:0.5><EOL>

Clauset et al 2007 equation 3.2:
    sigma = (alpha-1)/sqrt(n)


Plot xmin versus the ks value for derived alpha.  This plot can be used
as a diagnostic of whether you have derived the 'best' fit: if there are
multiple local minima, your data set may be well suited to a broken
powerlaw or a different function.


def most_likely_alpha(data, xmin, alpharange=(<NUM_LIT>,<NUM_LIT>), n_alpha=<NUM_LIT>):

alpha_vector = np.linspace(alpharange[<NUM_LIT:0>],alpharange[<NUM_LIT:1>],n_alpha)<EOL>return alpha_vector[discrete_max_likelihood_arg(data, xmin,<EOL>alpharange=alpharange,<EOL>n_alpha=n_alpha)]<EOL>

Return the most likely alpha for the data given an xmin


C = <NUM_LIT:1>/(-xm/(<NUM_LIT:1> - a) - xm/a + math.exp(a)*xm/a)<EOL>Ppl = lambda X: <NUM_LIT:1>+C*(xm/(<NUM_LIT:1>-a)*(X/xm)**(<NUM_LIT:1>-a))<EOL>Pexp = lambda X: C*xm/a*math.exp(a)-C*(xm/a)*math.exp(-a*(X/xm-<NUM_LIT:1>))<EOL>d=Ppl(x)<EOL>d[x<xm]=Pexp(x)<EOL>return d<EOL>

CDF(x) for the piecewise distribution exponential x<xmin, powerlaw x>=xmin
This is the CDF version of the distributions drawn in fig 3.4a of Clauset et al.


C = <NUM_LIT:1>/(-xm/(<NUM_LIT:1> - a) - xm/a + math.exp(a)*xm/a)<EOL>Pxm = <NUM_LIT:1>+C*(xm/(<NUM_LIT:1>-a))<EOL>pp = P<EOL>x = xm*(pp-<NUM_LIT:1>)*(<NUM_LIT:1>-a)/(C*xm)**(<NUM_LIT:1>/(<NUM_LIT:1>-a)) if pp >= Pxm else (math.log( ((C*xm/a)*math.exp(a)-pp)/(C*xm/a)) - a) * (-xm/a)<EOL>return x<EOL>

Inverse CDF for a piecewise PDF as defined in eqn. 3.10
of Clauset et al.


neg = [i<<NUM_LIT:0> for i in x]<EOL>if any(neg) > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (sum(neg)))<EOL>x = [i for i in x if i > <NUM_LIT:0>]<EOL><DEDENT>self.data = x<EOL>self.plfit(**kwargs)<EOL>


def alpha(xmin,x=x):<EOL><INDENT>"""<STR_LIT>"""<EOL>x = [i for i in x if i>=xmin]<EOL>n = sum(x)<EOL>divsum = sum([math.log(i/xmin) for i in x])<EOL>if divsum == <NUM_LIT:0>:<EOL><INDENT>return float('<STR_LIT>')<EOL><DEDENT>a = <NUM_LIT:1> + float(n) / divsum<EOL>return a<EOL><DEDENT>return alpha<EOL>

Create a mappable function alpha to apply to each xmin in a list of xmins.
        This is essentially the slow version of fplfit/cplfit, though I bet it could
        be speeded up with a clever use of parellel_map.  Not intended to be used by users.


xmins = unique(x)<EOL>xmins = xmins[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>dat = xmins * <NUM_LIT:0> <EOL>z = sort(x)<EOL>for xm in arange(len(xmins)):<EOL><INDENT>xmin = xmins[xm]<EOL>z    = z[z>=xmin] <EOL>n    = float(len(z))<EOL>a    =  n / sum( log(z/xmin) )<EOL>if nosmall:<EOL><INDENT>if (a-<NUM_LIT:1>)/sqrt(n) > <NUM_LIT:0.1>:<EOL><INDENT>dat = dat[:xm]<EOL>xm = len(xmins)+<NUM_LIT:1><EOL>break<EOL><DEDENT><DEDENT>cx   = arange(n)/float(n)  <EOL>cf   = <NUM_LIT:1>-(xmin/z)**a  <EOL>dat[xm] = max( abs(cf-cx) )<EOL><DEDENT>D     = min(dat);<EOL>xmin  = xmins[argmin(dat)]<EOL>z     = x[x>=xmin]<EOL>n     = len(z)<EOL>alpha = <NUM_LIT:1> + n / sum( log(z/xmin) )<EOL>if finite:<EOL><INDENT>alpha = alpha*(n-<NUM_LIT:1>)/n+<NUM_LIT:1>/n<EOL><DEDENT>if n < <NUM_LIT:50> and ~finite:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>L = n*log((alpha-<NUM_LIT:1>)/xmin) - alpha*sum(log(z/xmin));<EOL>return xmin,alpha,L,dat<EOL>

A Python implementation of the Matlab code http://www.santafe.edu/~aaronc/powerlaws/plfit.m
from http://www.santafe.edu/~aaronc/powerlaws/

See A. Clauset, C.R. Shalizi, and M.E.J. Newman, "Power-law distributions
in empirical data" SIAM Review, to appear (2009). (arXiv:0706.1062)
http://arxiv.org/abs/0706.1062


async def on_application_shutdown(self, application: Application):

Handler run at Application shutdown.

        This must be a coroutine.

        Subclasses can implement this to perform operations as part of
        Application.on_shutdown handler.


def _configure_registry(self, include_process_stats: bool = False):

if include_process_stats:<EOL><INDENT>self.registry.register_additional_collector(<EOL>ProcessCollector(registry=None))<EOL><DEDENT>


def _get_exporter(self, args: argparse.Namespace) -> PrometheusExporter:

exporter = PrometheusExporter(<EOL>self.name, self.description, args.host, args.port, self.registry)<EOL>exporter.app.on_startup.append(self.on_application_startup)<EOL>exporter.app.on_shutdown.append(self.on_application_shutdown)<EOL>return exporter<EOL>

Return a :class:`PrometheusExporter` configured with args.


def create_metrics(self,<EOL>configs: Iterable[MetricConfig]) -> Dict[str, Metric]:

metrics: Dict[str, Metric] = {<EOL>config.name: self._register_metric(config)<EOL>for config in configs<EOL>}<EOL>self._metrics.update(metrics)<EOL>return metrics<EOL>

Create Prometheus metrics from a list of MetricConfigs.


def get_metrics(self) -> Dict[str, Metric]:

return self._metrics.copy()<EOL>


run_app(<EOL>self.app,<EOL>host=self.host,<EOL>port=self.port,<EOL>print=lambda *args, **kargs: None,<EOL>access_log_format='<STR_LIT>')<EOL>

Run the :class:`aiohttp.web.Application` for the exporter.


def set_metric_update_handler(self, handler: UpdateHandler):

self._update_handler = handler<EOL>

Set a handler to update metrics.

        The provided coroutine function is called at every request with a dict
        as argument, mapping metric names to metrics.  The signature is the
        following:

          async def update_handler(metrics):