def get_last_user_medias(self, user_id, count):

return get_last_user_medias(self, user_id, count)<EOL>

Returns the last number of posts specified in count in media ids array.
:type count: int
:param count: Count of posts
:return: array


def like_user(self, user_id, amount=None, filtration=True):

if filtration:<EOL><INDENT>if not self.check_user(user_id):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>self.logger.info("<STR_LIT>" % user_id)<EOL>user_id = self.convert_to_user_id(user_id)<EOL>medias = self.get_user_medias(user_id, filtration=filtration)<EOL>if not medias:<EOL><INDENT>self.logger.info(<EOL>"<STR_LIT>")<EOL>return False<EOL><DEDENT>return self.like_medias(medias[:amount])<EOL>


def like_hashtag(self, hashtag, amount=None):

self.logger.info("<STR_LIT>" % hashtag)<EOL>medias = self.get_total_hashtag_medias(hashtag, amount)<EOL>return self.like_medias(medias)<EOL>


try:<EOL><INDENT>if not check_if_file_exists(file_path, quiet=quiet):<EOL><INDENT>return []<EOL><DEDENT>with codecs.open(file_path, "<STR_LIT:r>", encoding="<STR_LIT:utf-8>") as f:<EOL><INDENT>content = f.readlines()<EOL>if sys.version_info[<NUM_LIT:0>] < <NUM_LIT:3>:<EOL><INDENT>content = [str(item.encode('<STR_LIT:utf8>')) for item in content]<EOL><DEDENT>content = [item.strip() for item in content]<EOL>return [i for i in content if i]<EOL><DEDENT><DEDENT>except Exception as exception:<EOL><INDENT>print(str(exception))<EOL>return []<EOL><DEDENT>

Reads list from file. One line - one item.
Returns the list if file items.


while not check_secret():<EOL><INDENT>pass<EOL><DEDENT>while True:<EOL><INDENT>try:<EOL><INDENT>with open(SECRET_FILE, "<STR_LIT:r>") as f:<EOL><INDENT>lines = [line.strip().split("<STR_LIT::>", <NUM_LIT:2>) for line in f.readlines()]<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>msg = '<STR_LIT>'<EOL>raise Exception(msg.format(SECRET_FILE))<EOL><DEDENT>if username is not None:<EOL><INDENT>for login, password in lines:<EOL><INDENT>if login == username.strip():<EOL><INDENT>return login, password<EOL><DEDENT><DEDENT><DEDENT>print("<STR_LIT>")<EOL>for ind, (login, password) in enumerate(lines):<EOL><INDENT>print("<STR_LIT>" % (ind + <NUM_LIT:1>, login))<EOL><DEDENT>print("<STR_LIT>" % (<NUM_LIT:0>, "<STR_LIT>"))<EOL>print("<STR_LIT>" % (-<NUM_LIT:1>, "<STR_LIT>"))<EOL>try:<EOL><INDENT>ind = int(sys.stdin.readline())<EOL>if ind == <NUM_LIT:0>:<EOL><INDENT>add_credentials()<EOL>continue<EOL><DEDENT>elif ind == -<NUM_LIT:1>:<EOL><INDENT>delete_credentials()<EOL>check_secret()<EOL>continue<EOL><DEDENT>elif <NUM_LIT:0> <= ind - <NUM_LIT:1> < len(lines):<EOL><INDENT>return lines[ind - <NUM_LIT:1>]<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>

Returns login and password stored in `secret.txt`.


if data is None:<EOL><INDENT>data = {}<EOL><DEDENT>data.update(self.default_data)<EOL>return json.dumps(data)<EOL>

Adds the default_data to data and dumps it to a json.


return self.send_request('<STR_LIT>')<EOL>

Returns 8 medias from timeline feed of logged user.


url = '<STR_LIT>'<EOL>res = self.send_request(<EOL>url,<EOL>post=self.json_data({<EOL>'<STR_LIT>': [str(x) for x in user_ids]<EOL>})<EOL>)<EOL>if res:<EOL><INDENT>if "<STR_LIT>" in self.last_json:<EOL><INDENT>return self.last_json["<STR_LIT>"]<EOL><DEDENT>return []<EOL><DEDENT>return []<EOL>

Input: user_ids - a list of user_id
Output: dictionary: user_id - stories data.
Basically, for each user output the same as after self.get_user_reel


if not isinstance(reels, list):<EOL><INDENT>reels = [reels]<EOL><DEDENT>story_seen = {}<EOL>now = int(time.time())<EOL>for i, story in enumerate(sorted(reels, key=lambda m: m['<STR_LIT>'], reverse=True)):<EOL><INDENT>story_seen_at = now - min(i + <NUM_LIT:1> + random.randint(<NUM_LIT:0>, <NUM_LIT:2>), max(<NUM_LIT:0>, now - story['<STR_LIT>']))<EOL>story_seen[<EOL>'<STR_LIT>'.format(story['<STR_LIT:id>'], story['<STR_LIT:user>']['<STR_LIT>'])<EOL>] = [<EOL>'<STR_LIT>'.format(story['<STR_LIT>'], story_seen_at)<EOL>]<EOL><DEDENT>data = self.json_data({<EOL>'<STR_LIT>': story_seen,<EOL>'<STR_LIT>': self.token,<EOL>'<STR_LIT>': self.uuid,<EOL>'<STR_LIT>': self.user_id<EOL>})<EOL>data = self.generate_signature(data)<EOL>return self.session.post('<STR_LIT>' + '<STR_LIT>', data=data).ok<EOL>

Input - the list of reels jsons
They can be aquired by using get_users_reel() or get_user_reel() methods


url = '<STR_LIT>'<EOL>return self.send_request(url)<EOL>


if self.rps_schedule and self.instances_schedule:<EOL><INDENT>raise StepperConfigurationError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>elif self.rps_schedule:<EOL><INDENT>info.status.publish('<STR_LIT>', self.rps_schedule)<EOL>return lp.create(self.rps_schedule)<EOL><DEDENT>elif self.instances_schedule:<EOL><INDENT>info.status.publish('<STR_LIT>', self.instances_schedule)<EOL>return ip.create(self.instances_schedule)<EOL><DEDENT>else:<EOL><INDENT>self.instances_schedule = []<EOL>info.status.publish('<STR_LIT>', self.instances_schedule)<EOL>return ip.create(self.instances_schedule)<EOL><DEDENT>


af_readers = {<EOL>'<STR_LIT>': missile.AmmoFileReader,<EOL>'<STR_LIT>': missile.SlowLogReader,<EOL>'<STR_LIT>': missile.LineReader,<EOL>'<STR_LIT>': missile.UriReader,<EOL>'<STR_LIT>': missile.UriPostReader,<EOL>'<STR_LIT>': missile.AccessLogReader,<EOL>'<STR_LIT>': missile.CaseLineReader,<EOL>}<EOL>if self.uris and self.ammo_file:<EOL><INDENT>raise StepperConfigurationError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>elif self.uris:<EOL><INDENT>ammo_gen = missile.UriStyleGenerator(<EOL>self.uris, self.headers, http_ver=self.http_ver)<EOL><DEDENT>elif self.ammo_file:<EOL><INDENT>if self.ammo_type in af_readers:<EOL><INDENT>if self.ammo_type == '<STR_LIT>':<EOL><INDENT>opener = resource.get_opener(self.ammo_file)<EOL>with opener(self.use_cache) as ammo:<EOL><INDENT>try:<EOL><INDENT>if not ammo.next()[<NUM_LIT:0>].isdigit():<EOL><INDENT>self.ammo_type = '<STR_LIT>'<EOL>self.log.info(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>else:<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>self.log.exception(<EOL>"<STR_LIT>")<EOL>raise AmmoFileError(<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise NotImplementedError(<EOL>'<STR_LIT>' % self.ammo_type)<EOL><DEDENT>ammo_gen = af_readers[self.ammo_type](<EOL>self.ammo_file, headers=self.headers, http_ver=self.http_ver, use_cache=self.use_cache)<EOL><DEDENT>else:<EOL><INDENT>raise StepperConfigurationError(<EOL>'<STR_LIT>')<EOL><DEDENT>self.log.info("<STR_LIT>" % type(ammo_gen).__name__)<EOL>return ammo_gen<EOL>


_re_token = re.compile("<STR_LIT>")<EOL>def parse_token(time, multiplier):<EOL><INDENT>multipliers = {<EOL>'<STR_LIT:d>': <NUM_LIT>,<EOL>'<STR_LIT:h>': <NUM_LIT>,<EOL>'<STR_LIT:m>': <NUM_LIT>,<EOL>'<STR_LIT:s>': <NUM_LIT:1>,<EOL>}<EOL>if multiplier:<EOL><INDENT>if multiplier in multipliers:<EOL><INDENT>return int(float(time) * multipliers[multiplier] * <NUM_LIT:1000>)<EOL><DEDENT>else:<EOL><INDENT>raise StepperConfigurationError(<EOL>'<STR_LIT>' % duration)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return int(float(time) * <NUM_LIT:1000>)<EOL><DEDENT><DEDENT>return sum(parse_token(*token) for token in _re_token.findall(duration))<EOL>

Parse duration string, such as '3h2m3s' into milliseconds

>>> parse_duration('3h2m3s')
10923000

>>> parse_duration('0.3s')
300

>>> parse_duration('5')
5000


discRoot = math.sqrt((b * b) - <NUM_LIT:4> * a * c)<EOL>root1 = (-b - discRoot) / (<NUM_LIT:2> * a)<EOL>root2 = (-b + discRoot) / (<NUM_LIT:2> * a)<EOL>return (root1, root2)<EOL>


return int(n) + (n / abs(n)) * int(abs(n - int(n)) >= <NUM_LIT:0.5>) if n != <NUM_LIT:0> else <NUM_LIT:0><EOL>

rounds float to closest int
:rtype: int
:param n: float


if len(rps_schedule) > <NUM_LIT:1>:<EOL><INDENT>lp = Composite(<EOL>[StepFactory.produce(step_config) for step_config in rps_schedule])<EOL><DEDENT>else:<EOL><INDENT>lp = StepFactory.produce(rps_schedule[<NUM_LIT:0>])<EOL><DEDENT>info.status.publish('<STR_LIT>', lp.get_duration() / <NUM_LIT:1000>)<EOL>info.status.publish('<STR_LIT>', lp.get_rps_list())<EOL>info.status.lp_len = len(lp)<EOL>return lp<EOL>

Create Load Plan as defined in schedule. Publish info about its duration.


if <NUM_LIT:0> <= t <= self.duration:<EOL><INDENT>return self.rps<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>


return self.duration<EOL>


return int(self.duration / <NUM_LIT:1000> * self.rps)<EOL>


def __init__(self, minrps, maxrps, duration):

self.minrps = float(minrps)<EOL>self.maxrps = float(maxrps)<EOL>self.duration = duration / <NUM_LIT><EOL>self.slope = (self.maxrps - self.minrps) / self.duration<EOL>

:param minrps:
:param maxrps:
:param duration: milliseconds


try:<EOL><INDENT>root1, root2 = solve_quadratic(self.slope / <NUM_LIT>, self.minrps, -n)<EOL><DEDENT>except ZeroDivisionError:<EOL><INDENT>root2 = float(n) / self.minrps<EOL><DEDENT>return int(root2 * <NUM_LIT:1000>)<EOL>

:param n: number of charge
:return: when to shoot nth charge, milliseconds


return (self.ts(n) for n in range(<NUM_LIT:0>, self.__len__()))<EOL>


if <NUM_LIT:0> <= t <= self.duration:<EOL><INDENT>return self.minrps +float(self.maxrps - self.minrps) * t / self.duration<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>


return int(self.duration * <NUM_LIT:1000>)<EOL>


return int((self.maxrps + self.minrps) / <NUM_LIT> * self.duration)<EOL>


int_rps = range(int(self.minrps), int(self.maxrps) + <NUM_LIT:1>)<EOL>step_duration = float(self.duration) / len(int_rps)<EOL>rps_list = [(rps, int(step_duration)) for rps in int_rps]<EOL>return rps_list<EOL>

get list of constant load parts (we have no constant load at all, but tank will think so),
with parts durations (float)


seconds = range(<NUM_LIT:0>, int(self.duration) + <NUM_LIT:1>)<EOL>rps_groups = groupby([proper_round(self.rps_at(t)) for t in seconds],<EOL>lambda x: x)<EOL>rps_list = [(rps, len(list(rpl))) for rps, rpl in rps_groups]<EOL>return rps_list<EOL>

get list of each second's rps
:returns: list of tuples (rps, duration of corresponding rps in seconds)
:rtype: list


return sum(step.get_duration() for step in self.steps)<EOL>


return int(sum(step.__len__() for step in self.steps))<EOL>


try:<EOL><INDENT>limit = int(marker_type)<EOL>if limit:<EOL><INDENT>marker = __UriMarker(limit)<EOL><DEDENT>else:<EOL><INDENT>def marker(m):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT><DEDENT><DEDENT>except ValueError:<EOL><INDENT>if marker_type in __markers:<EOL><INDENT>marker = __markers[marker_type]<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError('<STR_LIT>' % marker_type)<EOL><DEDENT><DEDENT>if enum_ammo:<EOL><INDENT>marker = __Enumerator(marker)<EOL><DEDENT>return marker<EOL>

Returns a marker function of the requested marker_type

>>> marker = get_marker('uniq')(__test_missile)
>>> type(marker)
<type 'str'>
>>> len(marker)
32

>>> get_marker('uri')(__test_missile)
'_example_search_hello_help_us'

>>> marker = get_marker('non-existent')(__test_missile)
Traceback (most recent call last):
  ...
NotImplementedError: No such marker: "non-existent"

>>> get_marker('3')(__test_missile)
'_example_search_hello'

>>> marker = get_marker('3', True)
>>> marker(__test_missile)
'_example_search_hello#0'
>>> marker(__test_missile)
'_example_search_hello#1'


self.factory = factory<EOL>self.load_plan = factory.get_load_plan()<EOL>self.ammo_generator = factory.get_ammo_generator()<EOL>self.filter = factory.get_filter()<EOL>self.marker = factory.get_marker()<EOL>

Factory parameter is a configured ComponentFactory that
is able to produce load plan and ammo generator.


ammo_stream = (<EOL>ammo<EOL>for ammo in ((missile, marker or self.marker(missile))<EOL>for missile, marker in self.ammo_generator)<EOL>if self.filter(ammo))<EOL>return ((timestamp, marker or self.marker(missile), missile)<EOL>for timestamp, (missile, marker<EOL>) in zip(self.load_plan, ammo_stream))<EOL>

Returns a generator of (timestamp, marker, missile) tuples
where missile is in a string representation. Load Plan (timestamps
generator) and ammo generator are taken from the previously
configured ComponentFactory, passed as a parameter to the
__init__ method of this class.


def get_option(self, option, param2=None):

result = self.cfg[option]<EOL>self.log.debug(<EOL>"<STR_LIT>", option, result)<EOL>return result<EOL>


self.log.info("<STR_LIT>")<EOL>self.ammo_file = self.get_option(self.OPTION_AMMOFILE)<EOL>self.ammo_type = self.get_option('<STR_LIT>')<EOL>if self.ammo_file:<EOL><INDENT>self.ammo_file = os.path.expanduser(self.ammo_file)<EOL><DEDENT>self.loop_limit = self.get_option(self.OPTION_LOOP)<EOL>self.ammo_limit = self.get_option("<STR_LIT>")<EOL>self.load_profile = LoadProfile(**self.get_option('<STR_LIT>'))<EOL>self.instances = int(<EOL>self.get_option(self.OPTION_INSTANCES_LIMIT, '<STR_LIT>'))<EOL>self.uris = self.get_option("<STR_LIT>", [])<EOL>while '<STR_LIT>' in self.uris:<EOL><INDENT>self.uris.remove('<STR_LIT>')<EOL><DEDENT>self.headers = self.get_option("<STR_LIT>")<EOL>self.http_ver = self.get_option("<STR_LIT>")<EOL>self.autocases = self.get_option("<STR_LIT>")<EOL>self.enum_ammo = self.get_option("<STR_LIT>")<EOL>self.use_caching = self.get_option("<STR_LIT>")<EOL>self.file_cache = self.get_option('<STR_LIT>')<EOL>cache_dir = self.get_option("<STR_LIT>") or self.core.artifacts_base_dir<EOL>self.cache_dir = os.path.expanduser(cache_dir)<EOL>self.force_stepping = self.get_option("<STR_LIT>")<EOL>if self.get_option(self.OPTION_LOAD)[self.OPTION_LOAD_TYPE] == '<STR_LIT>':<EOL><INDENT>self.stpd = self.get_option(self.OPTION_LOAD)[self.OPTION_SCHEDULE]<EOL><DEDENT>self.chosen_cases = self.get_option("<STR_LIT>").split()<EOL>if self.chosen_cases:<EOL><INDENT>self.log.info("<STR_LIT>", self.chosen_cases)<EOL><DEDENT>


def publish_info(stepper_info):<EOL><INDENT>info.status.publish('<STR_LIT>', stepper_info.loadscheme)<EOL>info.status.publish('<STR_LIT>', stepper_info.loop_count)<EOL>info.status.publish('<STR_LIT>', stepper_info.steps)<EOL>info.status.publish('<STR_LIT>', stepper_info.duration)<EOL>info.status.ammo_count = stepper_info.ammo_count<EOL>info.status.publish('<STR_LIT>', stepper_info.instances)<EOL>self.core.publish('<STR_LIT>', '<STR_LIT>', stepper_info.loadscheme)<EOL>self.core.publish('<STR_LIT>', '<STR_LIT>', stepper_info.loop_count)<EOL>self.core.publish('<STR_LIT>', '<STR_LIT>', stepper_info.steps)<EOL>self.core.publish('<STR_LIT>', '<STR_LIT>', stepper_info.duration)<EOL>self.core.publish('<STR_LIT>', '<STR_LIT>', stepper_info.ammo_count)<EOL>self.core.publish('<STR_LIT>', '<STR_LIT>', stepper_info.instances)<EOL>return stepper_info<EOL><DEDENT>if not self.stpd:<EOL><INDENT>self.stpd = self.__get_stpd_filename()<EOL>if self.use_caching and not self.force_stepping and os.path.exists(<EOL>self.stpd) and os.path.exists(self.__si_filename()):<EOL><INDENT>self.log.info("<STR_LIT>", self.stpd)<EOL>stepper_info = self.__read_cached_options()<EOL>if self.instances and self.load_profile.is_rps():<EOL><INDENT>self.log.info(<EOL>"<STR_LIT>",<EOL>self.instances)<EOL>stepper_info = stepper_info._replace(<EOL>instances=self.instances)<EOL><DEDENT>publish_info(stepper_info)<EOL><DEDENT>else:<EOL><INDENT>if (<EOL>self.force_stepping and os.path.exists(self.__si_filename())):<EOL><INDENT>os.remove(self.__si_filename())<EOL><DEDENT>self.__make_stpd_file()<EOL>stepper_info = info.status.get_info()<EOL>self.__write_cached_options(stepper_info)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.log.info("<STR_LIT>", self.stpd)<EOL>stepper_info = publish_info(self.__read_cached_options())<EOL><DEDENT>self.ammo_count = stepper_info.ammo_count<EOL>self.duration = stepper_info.duration<EOL>self.loop_count = stepper_info.loop_count<EOL>self.loadscheme = stepper_info.loadscheme<EOL>self.steps = stepper_info.steps<EOL>if stepper_info.instances:<EOL><INDENT>self.instances = stepper_info.instances<EOL><DEDENT>


return "<STR_LIT>" % self.stpd<EOL>


if self.use_caching:<EOL><INDENT>sep = "<STR_LIT:|>"<EOL>hasher = hashlib.md5()<EOL>hashed_str = "<STR_LIT>" + sep +'<STR_LIT:;>'.join(self.load_profile.schedule) + sep + str(self.loop_limit)<EOL>hashed_str += sep + str(self.ammo_limit) + sep + '<STR_LIT:;>'.join(<EOL>self.load_profile.schedule) + sep + str(self.autocases)<EOL>hashed_str += sep + "<STR_LIT:;>".join(self.uris) + sep + "<STR_LIT:;>".join(<EOL>self.headers) + sep + self.http_ver + sep + "<STR_LIT:;>".join(<EOL>self.chosen_cases)<EOL>hashed_str += sep + str(self.enum_ammo) + sep + str(self.ammo_type)<EOL>if self.load_profile.is_instances():<EOL><INDENT>hashed_str += sep + str(self.instances)<EOL><DEDENT>if self.ammo_file:<EOL><INDENT>opener = resource.get_opener(self.ammo_file)<EOL>hashed_str += sep + opener.hash<EOL><DEDENT>else:<EOL><INDENT>if not self.uris:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>hashed_str += sep +'<STR_LIT:;>'.join(self.uris) + sep + '<STR_LIT:;>'.join(self.headers)<EOL><DEDENT>self.log.debug("<STR_LIT>", hashed_str)<EOL>hasher.update(hashed_str.encode('<STR_LIT:utf8>'))<EOL>if not os.path.exists(self.cache_dir):<EOL><INDENT>os.makedirs(self.cache_dir)<EOL><DEDENT>stpd = self.cache_dir + '<STR_LIT:/>' +os.path.basename(self.ammo_file) +"<STR_LIT:_>" + hasher.hexdigest() + "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>stpd = os.path.realpath("<STR_LIT>")<EOL><DEDENT>self.log.debug("<STR_LIT>", stpd)<EOL>return stpd<EOL>


self.log.debug("<STR_LIT>", self.__si_filename())<EOL>with open(self.__si_filename(), '<STR_LIT:r>') as si_file:<EOL><INDENT>si = info.StepperInfo(**json.load(si_file))<EOL><DEDENT>return si<EOL>


self.log.debug("<STR_LIT>", self.__si_filename())<EOL>with open(self.__si_filename(), '<STR_LIT:w>') as si_file:<EOL><INDENT>json.dump(si._asdict(), si_file, indent=<NUM_LIT:4>)<EOL><DEDENT>


self.log.info("<STR_LIT>", self.stpd)<EOL>stepper = Stepper(<EOL>self.core,<EOL>rps_schedule=self.load_profile.schedule if self.load_profile.is_rps() else None,<EOL>http_ver=self.http_ver,<EOL>ammo_file=self.ammo_file,<EOL>instances_schedule=self.load_profile.schedule if self.load_profile.is_instances() else None,<EOL>instances=self.instances,<EOL>loop_limit=self.loop_limit,<EOL>ammo_limit=self.ammo_limit,<EOL>uris=self.uris,<EOL>headers=[header.strip('<STR_LIT>') for header in self.headers],<EOL>autocases=self.autocases,<EOL>enum_ammo=self.enum_ammo,<EOL>ammo_type=self.ammo_type,<EOL>chosen_cases=self.chosen_cases,<EOL>use_cache=self.use_caching)<EOL>with open(self.stpd, '<STR_LIT:w>', self.file_cache) as os:<EOL><INDENT>stepper.write(os)<EOL><DEDENT>


lpb = LoadPlanBuilder().add_all_steps(instances_schedule)<EOL>lp = lpb.create()<EOL>info.status.publish('<STR_LIT>', <NUM_LIT:0>)<EOL>info.status.publish('<STR_LIT>', [])<EOL>info.status.publish('<STR_LIT>', lpb.instances)<EOL>return lp<EOL>


self.missiles = cycle([(missile_sample.to_s(), None)])<EOL>

Missile sample is any object that has to_s method which
returns its string representation.


def __init__(self, uris, headers, http_ver='<STR_LIT>'):

self.uri_count = len(uris)<EOL>self.missiles = cycle([(<EOL>HttpAmmo(<EOL>uri, headers, http_ver=http_ver).to_s(), None) for uri in uris])<EOL>


def init_logging(events_log_fname, verbose, quiet):

logger = logging.getLogger('<STR_LIT>')<EOL>logger.setLevel(logging.DEBUG)<EOL>console_handler = logging.StreamHandler(sys.stdout)<EOL>stderr_hdl = logging.StreamHandler(sys.stderr)<EOL>fmt_verbose = logging.Formatter(<EOL>"<STR_LIT>"<EOL>)<EOL>fmt_regular = logging.Formatter(<EOL>"<STR_LIT>", "<STR_LIT>")<EOL>if verbose:<EOL><INDENT>console_handler.setLevel(logging.DEBUG)<EOL>console_handler.setFormatter(fmt_verbose)<EOL>stderr_hdl.setFormatter(fmt_verbose)<EOL><DEDENT>elif quiet:<EOL><INDENT>console_handler.setLevel(logging.WARNING)<EOL>console_handler.setFormatter(fmt_regular)<EOL>stderr_hdl.setFormatter(fmt_regular)<EOL><DEDENT>else:<EOL><INDENT>console_handler.setLevel(logging.INFO)<EOL>console_handler.setFormatter(fmt_regular)<EOL>stderr_hdl.setFormatter(fmt_regular)<EOL><DEDENT>f_err = SingleLevelFilter(logging.ERROR, True)<EOL>f_warn = SingleLevelFilter(logging.WARNING, True)<EOL>f_crit = SingleLevelFilter(logging.CRITICAL, True)<EOL>console_handler.addFilter(f_err)<EOL>console_handler.addFilter(f_warn)<EOL>console_handler.addFilter(f_crit)<EOL>logger.addHandler(console_handler)<EOL>f_info = SingleLevelFilter(logging.INFO, True)<EOL>f_debug = SingleLevelFilter(logging.DEBUG, True)<EOL>stderr_hdl.addFilter(f_info)<EOL>stderr_hdl.addFilter(f_debug)<EOL>logger.addHandler(stderr_hdl)<EOL>if events_log_fname:<EOL><INDENT>err_file_handler = logging.FileHandler(events_log_fname)<EOL>err_file_handler.setLevel(logging.WARNING)<EOL>err_file_handler.setFormatter(<EOL>logging.Formatter(<EOL>"<STR_LIT>"<EOL>))<EOL>return [err_file_handler, console_handler, stderr_hdl]<EOL><DEDENT>else:<EOL><INDENT>return [console_handler, stderr_hdl]<EOL><DEDENT>

Set up logging, as it is very important for console tool


self.monitoring_plugins = monitoring_plugins<EOL>self.aggregator = aggregator<EOL>self.tank = tank<EOL>self._phantom_info = None<EOL>self.generator_plugin = generator_plugin<EOL>

:type aggregator: TankAggregator
:type monitoring_plugins: list of


def __init__(self, configs, interrupted_event, artifacts_base_dir=None, artifacts_dir_name=None):

self.output = {}<EOL>self.raw_configs = configs<EOL>self.status = {}<EOL>self._plugins = None<EOL>self._artifacts_dir = None<EOL>self.artifact_files = {}<EOL>self.artifacts_to_send = []<EOL>self._artifacts_base_dir = None<EOL>self.manual_start = False<EOL>self.scheduled_start = None<EOL>self.taskset_path = None<EOL>self.taskset_affinity = None<EOL>self._job = None<EOL>self._cfg_snapshot = None<EOL>self.interrupted = interrupted_event<EOL>self.error_log = None<EOL>error_output = '<STR_LIT>'<EOL>self.config, self.errors, self.configinitial = TankConfig(self.raw_configs,<EOL>with_dynamic_options=True,<EOL>core_section=self.SECTION,<EOL>error_output=error_output).validate()<EOL>if not self.config:<EOL><INDENT>raise ValidationError(self.errors)<EOL><DEDENT>self.test_id = self.get_option(self.SECTION, '<STR_LIT>',<EOL>datetime.datetime.now().strftime("<STR_LIT>"))<EOL>self.lock_dir = self.get_option(self.SECTION, '<STR_LIT>')<EOL>with open(os.path.join(self.artifacts_dir, CONFIGINITIAL), '<STR_LIT:w>') as f:<EOL><INDENT>yaml.dump(self.configinitial, f)<EOL><DEDENT>self.add_artifact_file(error_output)<EOL>self.add_artifact_to_send(LPRequisites.CONFIGINITIAL, yaml.dump(self.configinitial))<EOL>configinfo = self.config.validated.copy()<EOL>configinfo.setdefault(self.SECTION, {})<EOL>configinfo[self.SECTION][self.API_JOBNO] = self.test_id<EOL>self.add_artifact_to_send(LPRequisites.CONFIGINFO, yaml.dump(configinfo))<EOL>with open(os.path.join(self.artifacts_dir, VALIDATED_CONF), '<STR_LIT:w>') as f:<EOL><INDENT>yaml.dump(configinfo, f)<EOL><DEDENT>logger.info('<STR_LIT>' % self.test_id)<EOL>

:param configs: list of dict
:param interrupted_event: threading.Event


if self._plugins is None:<EOL><INDENT>self.load_plugins()<EOL>if self._plugins is None:<EOL><INDENT>self._plugins = {}<EOL><DEDENT><DEDENT>return self._plugins<EOL>

:returns: {plugin_name: plugin_class, ...}
:rtype: dict


logger.info("<STR_LIT>")<EOL>for (plugin_name, plugin_path, plugin_cfg) in self.config.plugins:<EOL><INDENT>logger.debug("<STR_LIT>", plugin_name, plugin_path)<EOL>if plugin_path == "<STR_LIT>":<EOL><INDENT>logger.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>plugin_path = "<STR_LIT>"<EOL><DEDENT>try:<EOL><INDENT>plugin = il.import_module(plugin_path)<EOL><DEDENT>except ImportError:<EOL><INDENT>logger.warning('<STR_LIT>', plugin_name, plugin_path)<EOL>logger.debug('<STR_LIT>', plugin_name, plugin_path, exc_info=True)<EOL>raise<EOL><DEDENT>try:<EOL><INDENT>instance = getattr(plugin, '<STR_LIT>')(self, cfg=plugin_cfg, name=plugin_name)<EOL><DEDENT>except AttributeError:<EOL><INDENT>logger.warning('<STR_LIT>', plugin_name)<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>self.register_plugin(self.PLUGIN_PREFIX + plugin_name, instance)<EOL><DEDENT><DEDENT>logger.debug("<STR_LIT>", self._plugins)<EOL>

Tells core to take plugin options and instantiate plugin classes


self.publish("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>logger.info("<STR_LIT>")<EOL>self.taskset_affinity = self.get_option(self.SECTION, '<STR_LIT>')<EOL>if self.taskset_affinity:<EOL><INDENT>self.__setup_taskset(self.taskset_affinity, pid=os.getpid())<EOL><DEDENT>for plugin in self.plugins.values():<EOL><INDENT>if not self.interrupted.is_set():<EOL><INDENT>logger.debug("<STR_LIT>", plugin)<EOL>plugin.configure()<EOL><DEDENT><DEDENT>


if not self.interrupted.is_set():<EOL><INDENT>logger.info("<STR_LIT>")<EOL>logger.info('<STR_LIT>'.format(dir=self.artifacts_dir))<EOL>self.publish("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>if not self.plugins:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>while not self.interrupted.is_set():<EOL><INDENT>begin_time = time.time()<EOL>aggr_retcode = self.job.aggregator.is_test_finished()<EOL>if aggr_retcode >= <NUM_LIT:0>:<EOL><INDENT>return aggr_retcode<EOL><DEDENT>for plugin in self.plugins.values():<EOL><INDENT>logger.debug("<STR_LIT>", plugin)<EOL>retcode = plugin.is_test_finished()<EOL>if retcode >= <NUM_LIT:0>:<EOL><INDENT>return retcode<EOL><DEDENT><DEDENT>end_time = time.time()<EOL>diff = end_time - begin_time<EOL>logger.debug("<STR_LIT>", diff)<EOL>logger.debug("<STR_LIT>", json.dumps(self.status))<EOL>if diff < <NUM_LIT:0.5>:<EOL><INDENT>time.sleep(<NUM_LIT:0.5> - diff)<EOL><DEDENT><DEDENT>return <NUM_LIT:1><EOL>

Call is_test_finished() on all plugins 'till one of them initiates exit


logger.info("<STR_LIT>")<EOL>self.publish("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>for plugin in self.plugins.values():<EOL><INDENT>logger.debug("<STR_LIT>", plugin)<EOL>try:<EOL><INDENT>logger.debug("<STR_LIT>", retcode)<EOL>retcode = plugin.post_process(retcode)<EOL>logger.debug("<STR_LIT>", retcode)<EOL><DEDENT>except Exception:  <EOL><INDENT>logger.error("<STR_LIT>", plugin, exc_info=True)<EOL>if not retcode:<EOL><INDENT>retcode = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return retcode<EOL>


def __setup_taskset(self, affinity, pid=None, args=None):

self.taskset_path = self.get_option(self.SECTION, '<STR_LIT>')<EOL>if args:<EOL><INDENT>return [self.taskset_path, '<STR_LIT:-c>', affinity] + args<EOL><DEDENT>if pid:<EOL><INDENT>args = "<STR_LIT>" % (self.taskset_path, affinity, pid)<EOL>retcode, stdout, stderr = execute(args, shell=True, poll_period=<NUM_LIT:0.1>, catch_out=True)<EOL>logger.debug('<STR_LIT>', pid, stdout)<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>logger.info("<STR_LIT>", str(pid), affinity)<EOL><DEDENT>else:<EOL><INDENT>logger.debug('<STR_LIT>', retcode)<EOL>raise KeyError(stderr)<EOL><DEDENT><DEDENT>

if pid specified: set process w/ pid `pid` CPU affinity to specified `affinity` core(s)
            if args specified: modify list of args for Popen to start w/ taskset w/ affinity `affinity`


def set_option(self, section, option, value):

raise NotImplementedError<EOL>


logger.debug("<STR_LIT>", plugin_class)<EOL>matches = [plugin for plugin in self.plugins.values() if isinstance(plugin, plugin_class)]<EOL>if matches:<EOL><INDENT>if len(matches) > <NUM_LIT:1>:<EOL><INDENT>logger.debug(<EOL>"<STR_LIT>",<EOL>plugin_class)<EOL><DEDENT>return matches[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % plugin_class)<EOL><DEDENT>

Retrieve a plugin of desired class, KeyError raised otherwise


logger.debug("<STR_LIT>", plugin_class)<EOL>matches = [plugin for plugin in self.plugins.values() if isinstance(plugin, plugin_class)]<EOL>if matches:<EOL><INDENT>return matches<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % plugin_class)<EOL><DEDENT>

Retrieve a list of plugins of desired class, KeyError raised otherwise


def __collect_file(self, filename, keep_original=False):

dest = self.artifacts_dir + '<STR_LIT:/>' + os.path.basename(filename)<EOL>logger.debug("<STR_LIT>", filename, dest)<EOL>if not filename or not os.path.exists(filename):<EOL><INDENT>logger.warning("<STR_LIT>", filename)<EOL>return<EOL><DEDENT>if os.path.exists(dest):<EOL><INDENT>logger.warning("<STR_LIT>", dest)<EOL>return<EOL><DEDENT>if keep_original:<EOL><INDENT>shutil.copy(filename, self.artifacts_dir)<EOL><DEDENT>else:<EOL><INDENT>shutil.move(filename, self.artifacts_dir)<EOL><DEDENT>os.chmod(dest, <NUM_LIT>)<EOL>

Move or copy single file to artifacts dir


def add_artifact_file(self, filename, keep_original=False):

if filename:<EOL><INDENT>logger.debug(<EOL>"<STR_LIT>", keep_original,<EOL>filename)<EOL>self.artifact_files[filename] = keep_original<EOL><DEDENT>

Add file to be stored as result artifact on post-process phase


def mkstemp(self, suffix, prefix, directory=None):

if not directory:<EOL><INDENT>directory = self.artifacts_dir<EOL><DEDENT>fd, fname = tempfile.mkstemp(suffix, prefix, directory)<EOL>os.close(fd)<EOL>os.chmod(fname, <NUM_LIT>)  <EOL>return fname<EOL>

Generate temp file name in artifacts base dir
and close temp file handle


logger.info("<STR_LIT>")<EOL>for plugin in self.plugins.values():<EOL><INDENT>logger.debug("<STR_LIT>", plugin)<EOL>try:<EOL><INDENT>plugin.close()<EOL><DEDENT>except Exception as ex:<EOL><INDENT>logger.error("<STR_LIT>", plugin, ex)<EOL>logger.debug(<EOL>"<STR_LIT>", traceback.format_exc(ex))<EOL><DEDENT><DEDENT>


logger.debug("<STR_LIT>", configs)<EOL>config_filenames = [resource.resource_filename(config) for config in configs]<EOL>try:<EOL><INDENT>self.config.read(config_filenames)<EOL><DEDENT>except Exception as ex:<EOL><INDENT>logger.error("<STR_LIT>", ex)<EOL>raise ex<EOL><DEDENT>


if not filename:<EOL><INDENT>filename = self.file<EOL><DEDENT>if filename:<EOL><INDENT>with open(filename, '<STR_LIT:w>') as handle:<EOL><INDENT>self.config.write(handle)<EOL><DEDENT><DEDENT>


def get_options(self, section, prefix='<STR_LIT>'):

res = []<EOL>try:<EOL><INDENT>for option in self.config.options(section):<EOL><INDENT>if not prefix or option.find(prefix) == <NUM_LIT:0>:<EOL><INDENT>res += [(<EOL>option[len(prefix):], self.config.get(section, option))]<EOL><DEDENT><DEDENT><DEDENT>except ConfigParser.NoSectionError as ex:<EOL><INDENT>logger.warning("<STR_LIT>", ex)<EOL><DEDENT>logger.debug(<EOL>"<STR_LIT>", section, prefix, res)<EOL>return res<EOL>


res = []<EOL>for section in self.config.sections():<EOL><INDENT>if section.startswith(prefix):<EOL><INDENT>res.append(section)<EOL><DEDENT><DEDENT>return res<EOL>


raise KeyboardInterrupt()<EOL>

required for non-tty python runs to interrupt


if is_ini(cfg_filename):<EOL><INDENT>return convert_ini(cfg_filename)<EOL><DEDENT>else:<EOL><INDENT>with open(cfg_filename) as f:<EOL><INDENT>return yaml.load(f)<EOL><DEDENT><DEDENT>


CFG_WILDCARD = '<STR_LIT>'<EOL>return [load_cfg(filename) for filename in sorted(glob.glob(os.path.join(path, CFG_WILDCARD)))]<EOL>


if options is None:<EOL><INDENT>return []<EOL><DEDENT>else:<EOL><INDENT>return [<EOL>convert_single_option(key.strip(), value.strip())<EOL>for key, value<EOL>in [option.split('<STR_LIT:=>', <NUM_LIT:1>) for option in options]<EOL>]<EOL><DEDENT>

:type options: list of str
:rtype: list of dict


def apply_shorthand_options(config, options, default_section='<STR_LIT>'):

if not options:<EOL><INDENT>return config<EOL><DEDENT>for option_str in options:<EOL><INDENT>key, value = option_str.split('<STR_LIT:=>')<EOL>try:<EOL><INDENT>section, option = key.split('<STR_LIT:.>')<EOL><DEDENT>except ValueError:<EOL><INDENT>section = default_section<EOL>option = key<EOL><DEDENT>if not config.has_section(section):<EOL><INDENT>config.add_section(section)<EOL><DEDENT>config.set(section, option, value)<EOL><DEDENT>return config<EOL>


<EOL>configs = [resource_filename(__name__, '<STR_LIT>')]<EOL>baseconfigs_location = '<STR_LIT>'<EOL>try:<EOL><INDENT>conf_files = sorted(os.listdir(baseconfigs_location))<EOL>for filename in conf_files:<EOL><INDENT>if fnmatch.fnmatch(filename, '<STR_LIT>'):<EOL><INDENT>configs += [<EOL>os.path.realpath(<EOL>baseconfigs_location + os.sep + filename)<EOL>]<EOL><DEDENT><DEDENT><DEDENT>except OSError:<EOL><INDENT>logger.info(<EOL>baseconfigs_location + '<STR_LIT>')<EOL><DEDENT>configs += [os.path.expanduser('<STR_LIT>')]<EOL>return configs<EOL>

returns default configs list, from /etc and home dir


for val in [<EOL>self.YELLOW, self.RED, self.RESET, self.CYAN, self.BG_MAGENTA,<EOL>self.WHITE, self.BG_GREEN, self.GREEN, self.BG_BROWN,<EOL>self.RED_DARK, self.MAGENTA, self.BG_CYAN<EOL>]:<EOL><INDENT>orig_str = orig_str.replace(val, '<STR_LIT>')<EOL><DEDENT>return orig_str<EOL>


self._actions = []<EOL>self.tankworker = tankworker<EOL>


assert callable(fn)<EOL>self._actions.append((name, fn))<EOL>


self.worker = tankworker<EOL>


self.log.info("<STR_LIT>", cmd)<EOL>retcode = execute(<EOL>cmd, shell=True, poll_period=<NUM_LIT:0.1>, catch_out=self.catch_out)[<NUM_LIT:0>]<EOL>if retcode:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % retcode)<EOL><DEDENT>return retcode<EOL>


This is a module initialization function.


return not self.workers_finished<EOL>

True while there are alive workers out there. Tank
will quit when this would become False


self.quit.set()<EOL>while sorted([<EOL>self.pool[i].is_alive()<EOL>for i in xrange(len(self.pool))])[-<NUM_LIT:1>]:<EOL><INDENT>time.sleep(<NUM_LIT:1>)<EOL><DEDENT>try:<EOL><INDENT>while not self.task_queue.empty():<EOL><INDENT>self.task_queue.get(timeout=<NUM_LIT:0.1>)<EOL><DEDENT>self.task_queue.close()<EOL>self.feeder.join()<EOL><DEDENT>except Exception as ex:<EOL><INDENT>logger.info(ex)<EOL><DEDENT>

Say the workers to finish their jobs and quit.


self.plan = StpdReader(self.stpd_filename)<EOL>if self.cached_stpd:<EOL><INDENT>self.plan = list(self.plan)<EOL><DEDENT>for task in self.plan:<EOL><INDENT>if self.quit.is_set():<EOL><INDENT>logger.info("<STR_LIT>")<EOL>return<EOL><DEDENT>while True:<EOL><INDENT>try:<EOL><INDENT>self.task_queue.put(task, timeout=<NUM_LIT:1>)<EOL>break<EOL><DEDENT>except Full:<EOL><INDENT>if self.quit.is_set() or self.workers_finished:<EOL><INDENT>return<EOL><DEDENT>else:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT><DEDENT>workers_count = self.instances<EOL>logger.info(<EOL>"<STR_LIT>" % (workers_count))<EOL>retry_delay = <NUM_LIT:1><EOL>for _ in range(<NUM_LIT:5>):<EOL><INDENT>try:<EOL><INDENT>[<EOL>self.task_queue.put(None, timeout=<NUM_LIT:1>)<EOL>for _ in xrange(<NUM_LIT:0>, workers_count)<EOL>]<EOL>break<EOL><DEDENT>except Full:<EOL><INDENT>logger.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", retry_delay)<EOL>time.sleep(retry_delay)<EOL>retry_delay *= <NUM_LIT:2><EOL><DEDENT><DEDENT>try:<EOL><INDENT>logger.info("<STR_LIT>")<EOL>map(lambda x: x.join(), self.pool)<EOL>logger.info("<STR_LIT>")<EOL>self.workers_finished = True<EOL><DEDENT>except (KeyboardInterrupt, SystemExit):<EOL><INDENT>self.task_queue.close()<EOL>self.results.close()<EOL>self.quit.set()<EOL>logger.info("<STR_LIT>")<EOL>map(lambda x: x.join(), self.pool)<EOL>self.workers_finished = True<EOL><DEDENT>

A feeder that runs in distinct thread in main process.


logger.debug("<STR_LIT>")<EOL>try:<EOL><INDENT>self.gun.setup()<EOL><DEDENT>except Exception:<EOL><INDENT>logger.exception("<STR_LIT>")<EOL>return<EOL><DEDENT>while not self.quit.is_set():<EOL><INDENT>try:<EOL><INDENT>task = self.task_queue.get(timeout=<NUM_LIT:1>)<EOL>if not task:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>break<EOL><DEDENT>timestamp, missile, marker = task<EOL>planned_time = self.start_time + (timestamp / <NUM_LIT>)<EOL>delay = planned_time - time.time()<EOL>if delay > <NUM_LIT:0>:<EOL><INDENT>time.sleep(delay)<EOL><DEDENT>try:<EOL><INDENT>with self.instance_counter.get_lock():<EOL><INDENT>self.instance_counter.value += <NUM_LIT:1><EOL><DEDENT>self.gun.shoot(missile, marker)<EOL><DEDENT>finally:<EOL><INDENT>with self.instance_counter.get_lock():<EOL><INDENT>self.instance_counter.value -= <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>except (KeyboardInterrupt, SystemExit):<EOL><INDENT>break<EOL><DEDENT>except Empty:<EOL><INDENT>if self.quit.is_set():<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT><DEDENT>except Full:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT>except Exception:<EOL><INDENT>logger.exception("<STR_LIT>")<EOL><DEDENT><DEDENT>try:<EOL><INDENT>self.gun.teardown()<EOL><DEDENT>except Exception:<EOL><INDENT>logger.exception("<STR_LIT>")<EOL>return<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>


while not self.quit.is_set():<EOL><INDENT>try:<EOL><INDENT>task = self.green_queue.get(timeout=<NUM_LIT:1>)<EOL>timestamp, missile, marker = task<EOL>planned_time = self.start_time + (timestamp / <NUM_LIT>)<EOL>delay = planned_time - time.time()<EOL>if delay > <NUM_LIT:0>:<EOL><INDENT>time.sleep(delay)<EOL><DEDENT>try:<EOL><INDENT>with self.instance_counter.get_lock():<EOL><INDENT>self.instance_counter.value += <NUM_LIT:1><EOL><DEDENT>self.gun.shoot(missile, marker)<EOL><DEDENT>finally:<EOL><INDENT>with self.instance_counter.get_lock():<EOL><INDENT>self.instance_counter.value -= <NUM_LIT:1><EOL><DEDENT>self._free_threads_count += <NUM_LIT:1><EOL><DEDENT><DEDENT>except (KeyboardInterrupt, SystemExit):<EOL><INDENT>break<EOL><DEDENT>except Empty:<EOL><INDENT>continue<EOL><DEDENT>except Full:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT>except Exception:<EOL><INDENT>logger.exception("<STR_LIT>")<EOL><DEDENT><DEDENT>


if self.is_relative:<EOL><INDENT>level_str = str(<NUM_LIT:100> * self.level) + "<STR_LIT:%>"<EOL><DEDENT>else:<EOL><INDENT>level_str = self.level<EOL><DEDENT>return level_str<EOL>


if self.is_relative:<EOL><INDENT>level_str = str(self.level) + "<STR_LIT:%>"<EOL><DEDENT>else:<EOL><INDENT>level_str = self.level<EOL><DEDENT>return level_str<EOL>


if self.is_relative:<EOL><INDENT>level_str = str(self.level) + "<STR_LIT:%>"<EOL><DEDENT>else:<EOL><INDENT>level_str = str(self.level)<EOL><DEDENT>return level_str<EOL>


if len(tangents) < <NUM_LIT:2>:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>avg_tan = float(sum(tangents) / len(tangents))<EOL>numerator = float()<EOL>for i in tangents:<EOL><INDENT>numerator += (i - avg_tan) * (i - avg_tan)<EOL><DEDENT>return math.sqrt(numerator / len(tangents) / (len(tangents) - <NUM_LIT:1>))<EOL>

formula for measurement error
sqrt ( (sum(1, n, (k_i - <k>)**2) / (n*(n-1)))


return self.counting<EOL>


self.counting += [obj]<EOL>


self.custom_criterions += [criterion_class]<EOL>


parsed = criterion_str.split("<STR_LIT:(>")<EOL>type_str = parsed[<NUM_LIT:0>].strip().lower()<EOL>parsed[<NUM_LIT:1>] = parsed[<NUM_LIT:1>].split("<STR_LIT:)>")[<NUM_LIT:0>].strip()<EOL>for criterion_class in self.custom_criterions:<EOL><INDENT>if criterion_class.get_type_string() == type_str:<EOL><INDENT>return criterion_class(self, parsed[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>raise ValueError(<EOL>"<STR_LIT>" % criterion_str)<EOL>

instantiate criterion from config string


if len(param1) <= <NUM_LIT:3>:<EOL><INDENT>if success:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT><EOL><DEDENT><DEDENT>exc = param1.split('<STR_LIT:U+0020>')[-<NUM_LIT:1>]<EOL>if exc in KNOWN_EXC.keys():<EOL><INDENT>return KNOWN_EXC[exc]<EOL><DEDENT>else:<EOL><INDENT>logger.warning(<EOL>"<STR_LIT>",<EOL>param1)<EOL>return <NUM_LIT><EOL><DEDENT>

translate http code to net code. if accertion failed, set net code to 314


if len(param1) <= <NUM_LIT:3>:<EOL><INDENT>try:<EOL><INDENT>int(param1)<EOL><DEDENT>except BaseException:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>", param1)<EOL><DEDENT>else:<EOL><INDENT>return int(param1)<EOL><DEDENT><DEDENT>exc = param1.split('<STR_LIT:U+0020>')[-<NUM_LIT:1>]<EOL>if exc in KNOWN_EXC.keys():<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>logger.warning("<STR_LIT>", param1)<EOL>return <NUM_LIT:0><EOL><DEDENT>


r = re.compile(self.DISCOVER_PORT_PATTERN)<EOL>with open(self.process_stderr.name, '<STR_LIT:r>') as f:<EOL><INDENT>cnt = <NUM_LIT:0><EOL>while self.process.pid and cnt < <NUM_LIT:10>:<EOL><INDENT>line = f.readline()<EOL>m = r.match(line)<EOL>if m is None:<EOL><INDENT>cnt += <NUM_LIT:1><EOL>time.sleep(<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>port = int(m.group('<STR_LIT:port>'))<EOL>return port<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logger.warning('<STR_LIT>')<EOL>return None<EOL><DEDENT><DEDENT>

Searching for line in jmeter.log such as
        Waiting for possible shutdown message on port 4445


def __add_jmeter_components(self, jmx, jtl, variables):

logger.debug("<STR_LIT>", os.path.realpath(jmx))<EOL>with open(jmx, '<STR_LIT:r>') as src_jmx:<EOL><INDENT>source_lines = src_jmx.readlines()<EOL><DEDENT>try:<EOL><INDENT>closing = source_lines.pop(-<NUM_LIT:1>)<EOL>if "<STR_LIT>" in source_lines[-<NUM_LIT:5>]:<EOL><INDENT>logger.info("<STR_LIT>")<EOL>last_string_count = <NUM_LIT:6><EOL><DEDENT>else:<EOL><INDENT>last_string_count = <NUM_LIT:2><EOL><DEDENT>while last_string_count > <NUM_LIT:0>:<EOL><INDENT>closing = source_lines.pop(-<NUM_LIT:1>) + closing<EOL>last_string_count -= <NUM_LIT:1><EOL><DEDENT>logger.debug("<STR_LIT>", closing)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % exc)<EOL><DEDENT>udv_tpl = resource_string(__name__, '<STR_LIT>')<EOL>udv_set = []<EOL>for var_name, var_value in variables.iteritems():<EOL><INDENT>udv_set.append(udv_tpl % (var_name, var_name, var_value))<EOL><DEDENT>udv = "<STR_LIT:\n>".join(udv_set)<EOL>if self.jmeter_ver >= <NUM_LIT>:<EOL><INDENT>save_connect = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>save_connect = '<STR_LIT>'<EOL><DEDENT>if self.ext_log in ['<STR_LIT>', '<STR_LIT:all>']:<EOL><INDENT>level_map = {'<STR_LIT>': '<STR_LIT:true>', '<STR_LIT:all>': '<STR_LIT:false>'}<EOL>tpl_resource = '<STR_LIT>'<EOL>tpl_args = {<EOL>'<STR_LIT>': self.jtl_file,<EOL>'<STR_LIT>': udv,<EOL>'<STR_LIT>': self.ext_log_file,<EOL>'<STR_LIT>': level_map[self.ext_log],<EOL>'<STR_LIT>': save_connect<EOL>}<EOL><DEDENT>else:<EOL><INDENT>tpl_resource = '<STR_LIT>'<EOL>tpl_args = {<EOL>'<STR_LIT>': self.jtl_file,<EOL>'<STR_LIT>': udv,<EOL>'<STR_LIT>': save_connect<EOL>}<EOL><DEDENT>tpl = resource_string(__name__, '<STR_LIT>' + tpl_resource)<EOL>try:<EOL><INDENT>new_jmx = self.core.mkstemp(<EOL>'<STR_LIT>', '<STR_LIT>', os.path.dirname(os.path.realpath(jmx)))<EOL><DEDENT>except OSError as exc:<EOL><INDENT>logger.debug("<STR_LIT>", exc)<EOL>new_jmx = self.core.mkstemp('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>logger.debug("<STR_LIT>", new_jmx)<EOL>with open(new_jmx, "<STR_LIT:wb>") as fh:<EOL><INDENT>fh.write('<STR_LIT>'.join(source_lines))<EOL>fh.write(tpl % tpl_args)<EOL>fh.write(closing)<EOL><DEDENT>return new_jmx<EOL>


AbstractPlugin.__init__(self, core, cfg, name)<EOL>self.interval = "<STR_LIT>"<EOL>self.disk_limit = <NUM_LIT>  <EOL>self.mem_limit = <NUM_LIT>  <EOL>self.last_check = <NUM_LIT:0><EOL>


cmd = "<STR_LIT>"<EOL>cmd += self.core.artifacts_base_dir<EOL>cmd += "<STR_LIT>"<EOL>res = execute(cmd, True, <NUM_LIT:0.1>, True)<EOL>logging.debug("<STR_LIT>", res)<EOL>if not len(res[<NUM_LIT:1>]):<EOL><INDENT>self.log.debug("<STR_LIT>", res[<NUM_LIT:2>])<EOL>return<EOL><DEDENT>disk_free = res[<NUM_LIT:1>]<EOL>self.log.debug(<EOL>"<STR_LIT>", disk_free.strip(), self.disk_limit)<EOL>if int(disk_free.strip()) < self.disk_limit:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>% (<EOL>self.disk_limit, self.core.artifacts_base_dir,<EOL>int(disk_free.strip())))<EOL><DEDENT>


mem_free = psutil.virtual_memory().available / <NUM_LIT:2>**<NUM_LIT:20><EOL>self.log.debug("<STR_LIT>", mem_free, self.mem_limit)<EOL>if mem_free < self.mem_limit:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (self.mem_limit, mem_free))<EOL><DEDENT>


for date_str, statistics in chunk.iteritems():<EOL><INDENT>date_obj = datetime.datetime.strptime(<EOL>date_str.split("<STR_LIT:.>")[<NUM_LIT:0>], '<STR_LIT>')<EOL>chunk_date = int(time.mktime(date_obj.timetuple()))<EOL>instances = <NUM_LIT:0><EOL>for benchmark_name, benchmark in statistics.iteritems():<EOL><INDENT>if not benchmark_name.startswith("<STR_LIT>"):<EOL><INDENT>continue<EOL><DEDENT>for method, meth_obj in benchmark.iteritems():<EOL><INDENT>if "<STR_LIT>" in meth_obj:<EOL><INDENT>instances += meth_obj["<STR_LIT>"][<NUM_LIT:2>]<EOL><DEDENT><DEDENT><DEDENT>offset = chunk_date - <NUM_LIT:1> - self.start_time<EOL>reqps = <NUM_LIT:0><EOL>if <NUM_LIT:0> <= offset < len(self.phantom_info.steps):<EOL><INDENT>reqps = self.phantom_info.steps[offset][<NUM_LIT:0>]<EOL><DEDENT>yield self.stats_item(chunk_date - <NUM_LIT:1>, instances, reqps)<EOL><DEDENT>


self.start_time = int(time.time())<EOL>with open(self.stat_filename, '<STR_LIT:r>') as stat_file:<EOL><INDENT>while not self.closed:<EOL><INDENT>yield self._read_stat_data(stat_file)<EOL><DEDENT>data = self._read_stat_data(stat_file)<EOL>while data:<EOL><INDENT>yield data<EOL>data = self._read_stat_data(stat_file)<EOL><DEDENT><DEDENT>

Union buffer and chunk, split using '\n},',
return splitted parts


self.threads = self.cfg["<STR_LIT>"] or str(int(multiprocessing.cpu_count() / <NUM_LIT:2>) + <NUM_LIT:1>)<EOL>self.phantom_modules_path = self.cfg["<STR_LIT>"]<EOL>self.additional_libs = '<STR_LIT:U+0020>'.join(self.cfg["<STR_LIT>"])<EOL>self.answ_log_level = self.cfg["<STR_LIT>"]<EOL>if self.answ_log_level.lower() in ['<STR_LIT:0>', '<STR_LIT:false>']:<EOL><INDENT>self.answ_log_level = '<STR_LIT:none>'<EOL><DEDENT>elif self.answ_log_level.lower() in ['<STR_LIT:1>', '<STR_LIT:true>']:<EOL><INDENT>self.answ_log_level = '<STR_LIT:all>'<EOL><DEDENT>self.timeout = parse_duration(self.cfg["<STR_LIT>"])<EOL>if self.timeout > <NUM_LIT>:<EOL><INDENT>logger.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self.answ_log = self.core.mkstemp("<STR_LIT>", "<STR_LIT>")<EOL>self.core.add_artifact_file(self.answ_log)<EOL>self.core.add_artifact_file(self.phout_file)<EOL>self.core.add_artifact_file(self.stat_log)<EOL>self.phantom_log = self.core.mkstemp("<STR_LIT>", "<STR_LIT>")<EOL>self.core.add_artifact_file(self.phantom_log)<EOL>main_stream = StreamConfig(<EOL>self.core,<EOL>len(self.streams), self.phout_file, self.answ_log,<EOL>self.answ_log_level, self.timeout, self.cfg, True)<EOL>self.streams.append(main_stream)<EOL>for section in self.multi():<EOL><INDENT>self.streams.append(<EOL>StreamConfig(<EOL>self.core,<EOL>len(self.streams), self.phout_file, self.answ_log,<EOL>self.answ_log_level, self.timeout, section))<EOL><DEDENT>for stream in self.streams:<EOL><INDENT>stream.read_config()<EOL><DEDENT>if any(stream.ssl for stream in self.streams):<EOL><INDENT>self.additional_libs += '<STR_LIT>'<EOL><DEDENT>


streams_config = '<STR_LIT>'<EOL>stat_benchmarks = '<STR_LIT>'<EOL>for stream in self.streams:<EOL><INDENT>streams_config += stream.compose_config()<EOL>if not stream.is_main:<EOL><INDENT>stat_benchmarks += "<STR_LIT:U+0020>" + "<STR_LIT>" % stream.sequence_no<EOL><DEDENT><DEDENT>kwargs = {}<EOL>kwargs['<STR_LIT>'] = self.threads<EOL>kwargs['<STR_LIT>'] = self.phantom_log<EOL>kwargs['<STR_LIT>'] = self.stat_log<EOL>kwargs['<STR_LIT>'] = streams_config<EOL>kwargs['<STR_LIT>'] = stat_benchmarks<EOL>kwargs['<STR_LIT>'] = self.additional_libs<EOL>kwargs['<STR_LIT>'] = self.phantom_modules_path<EOL>filename = self.core.mkstemp("<STR_LIT>", "<STR_LIT>")<EOL>self.core.add_artifact_file(filename)<EOL>logger.debug("<STR_LIT>", filename)<EOL>template_str = resource_string(__name__, "<STR_LIT>")<EOL>tpl = string.Template(template_str)<EOL>config = tpl.substitute(kwargs)<EOL>with open(filename, '<STR_LIT:w>') as conffile:<EOL><INDENT>conffile.write(config)<EOL><DEDENT>return filename<EOL>


for stream in self.streams:<EOL><INDENT>stream.timeout = timeout<EOL><DEDENT>


result = copy.copy(self.streams[<NUM_LIT:0>])<EOL>result.stat_log = self.stat_log<EOL>result.steps = []<EOL>result.ammo_file = '<STR_LIT>'<EOL>result.rps_schedule = None<EOL>result.ammo_count = <NUM_LIT:0><EOL>result.duration = <NUM_LIT:0><EOL>result.instances = <NUM_LIT:0><EOL>result.loadscheme = []<EOL>result.loop_count = <NUM_LIT:0><EOL>for stream in self.streams:<EOL><INDENT>sec_no = <NUM_LIT:0><EOL>logger.debug("<STR_LIT>", stream.stepper_wrapper.steps)<EOL>for item in stream.stepper_wrapper.steps:<EOL><INDENT>for x in range(<NUM_LIT:0>, item[<NUM_LIT:1>]):<EOL><INDENT>if len(result.steps) > sec_no:<EOL><INDENT>result.steps[sec_no][<NUM_LIT:0>] += item[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>result.steps.append([item[<NUM_LIT:0>], <NUM_LIT:1>])<EOL><DEDENT>sec_no += <NUM_LIT:1><EOL><DEDENT><DEDENT>if result.rps_schedule:<EOL><INDENT>result.rps_schedule = []<EOL><DEDENT>else:<EOL><INDENT>result.rps_schedule = stream.stepper_wrapper.loadscheme<EOL><DEDENT>if result.loadscheme:<EOL><INDENT>result.loadscheme = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>result.loadscheme = '<STR_LIT>'<EOL><DEDENT>if result.loop_count:<EOL><INDENT>result.loop_count = u'<STR_LIT:0>'<EOL><DEDENT>else:<EOL><INDENT>result.loop_count = stream.stepper_wrapper.loop_count<EOL><DEDENT>result.ammo_file += '<STR_LIT>'.format(stream.stepper_wrapper.ammo_file)<EOL>result.ammo_count += stream.stepper_wrapper.ammo_count<EOL>result.duration = max(<EOL>result.duration, stream.stepper_wrapper.duration)<EOL>result.instances += stream.instances<EOL><DEDENT>if not result.ammo_count:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return result<EOL>


def get_option(self, option, default=None):

return self.cfg[option]<EOL>


<EOL>self.ssl = self.get_option("<STR_LIT>")<EOL>self.tank_type = self.get_option("<STR_LIT>")<EOL>self.gatling = '<STR_LIT:U+0020>'.join(self.get_option('<STR_LIT>').split("<STR_LIT:\n>"))<EOL>self.method_prefix = self.get_option("<STR_LIT>")<EOL>self.method_options = self.get_option("<STR_LIT>")<EOL>self.source_log_prefix = self.get_option("<STR_LIT>")<EOL>self.phantom_http_line = self.get_option("<STR_LIT>")<EOL>self.phantom_http_field_num = self.get_option("<STR_LIT>")<EOL>self.phantom_http_field = self.get_option("<STR_LIT>")<EOL>self.phantom_http_entity = self.get_option("<STR_LIT>")<EOL>self.address = self.get_option('<STR_LIT:address>')<EOL>do_test_connect = self.get_option("<STR_LIT>")<EOL>explicit_port = self.get_option('<STR_LIT:port>', '<STR_LIT>')<EOL>self.ipv6, self.resolved_ip, self.port, self.address = self.address_wizard.resolve(<EOL>self.address, do_test_connect, explicit_port)<EOL>logger.info(<EOL>"<STR_LIT>", self.address, self.resolved_ip, self.port)<EOL>self.client_cipher_suites = self.get_option("<STR_LIT>", "<STR_LIT>")<EOL>self.client_certificate = self.get_option("<STR_LIT>", "<STR_LIT>")<EOL>self.client_key = self.get_option("<STR_LIT>", "<STR_LIT>")<EOL>self.stepper_wrapper.read_config()<EOL>


<EOL>self.stepper_wrapper.prepare_stepper()<EOL>self.stpd = self.stepper_wrapper.stpd<EOL>if self.stepper_wrapper.instances:<EOL><INDENT>self.instances = self.stepper_wrapper.instances<EOL><DEDENT>if not self.stpd:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>kwargs = {}<EOL>kwargs['<STR_LIT>'] = self.sequence_no<EOL>if self.ssl:<EOL><INDENT>_auth_section = '<STR_LIT>'<EOL>_ciphers = '<STR_LIT>'<EOL>ssl_template = "<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>if self.client_certificate or self.client_key:<EOL><INDENT>_auth_section = '<STR_LIT>'% (self.client_key, self.client_certificate)<EOL><DEDENT>if self.client_cipher_suites:<EOL><INDENT>_ciphers = '<STR_LIT>' % self.client_cipher_suites<EOL><DEDENT>kwargs['<STR_LIT>'] = ssl_template % (_auth_section, _ciphers)<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>kwargs['<STR_LIT>'] = self.method_prefix +"<STR_LIT>" if self.ipv6 else self.method_prefix + "<STR_LIT>"<EOL>kwargs['<STR_LIT>'] = self.phout_file<EOL>kwargs['<STR_LIT>'] = self.answ_log<EOL>kwargs['<STR_LIT>'] = self.answ_log_level<EOL>kwargs['<STR_LIT>'] = "<STR_LIT>" if self.answ_log_level == '<STR_LIT:none>' else '<STR_LIT>'<EOL>kwargs['<STR_LIT>'] = self.stpd<EOL>kwargs['<STR_LIT>'] = self.source_log_prefix<EOL>kwargs['<STR_LIT>'] = self.method_options<EOL>if self.tank_type:<EOL><INDENT>kwargs[<EOL>'<STR_LIT>'] = "<STR_LIT>" % self.sequence_no if self.tank_type == '<STR_LIT:http>' else "<STR_LIT>"<EOL>kwargs['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT>'] = "<STR_LIT>"<EOL>kwargs['<STR_LIT>'] = "<STR_LIT:#>"<EOL><DEDENT>if self.gatling:<EOL><INDENT>kwargs['<STR_LIT>'] = '<STR_LIT>' + self.gatling + '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>kwargs['<STR_LIT>'] = self.resolved_ip<EOL>kwargs['<STR_LIT:port>'] = self.port<EOL>kwargs['<STR_LIT>'] = self.timeout<EOL>kwargs['<STR_LIT>'] = self.instances<EOL>tune = '<STR_LIT>'<EOL>if self.phantom_http_entity:<EOL><INDENT>tune += "<STR_LIT>" + self.phantom_http_entity + "<STR_LIT:\n>"<EOL><DEDENT>if self.phantom_http_field:<EOL><INDENT>tune += "<STR_LIT>" + self.phantom_http_field + "<STR_LIT:\n>"<EOL><DEDENT>if self.phantom_http_field_num:<EOL><INDENT>tune += "<STR_LIT>".format(self.phantom_http_field_num)<EOL><DEDENT>if self.phantom_http_line:<EOL><INDENT>tune += "<STR_LIT>" + self.phantom_http_line + "<STR_LIT:\n>"<EOL><DEDENT>if tune:<EOL><INDENT>kwargs['<STR_LIT>'] = '<STR_LIT>' + tune + "<STR_LIT:}>"<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if self.is_main:<EOL><INDENT>fname = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>fname = '<STR_LIT>'<EOL><DEDENT>template_str = resource_string(<EOL>__name__, "<STR_LIT>" + fname)<EOL>tpl = string.Template(template_str)<EOL>config = tpl.substitute(kwargs)<EOL>return config<EOL>


if not self._phantom:<EOL><INDENT>self._phantom = PhantomConfig(self.core, self.cfg, self.stat_log)<EOL>self._phantom.read_config()<EOL><DEDENT>return self._phantom<EOL>


if not self.cached_info:<EOL><INDENT>if not self.phantom:<EOL><INDENT>return None<EOL><DEDENT>self.cached_info = self.phantom.get_info()<EOL><DEDENT>return self.cached_info<EOL>


if self.is_relative:<EOL><INDENT>level_str = str(<NUM_LIT:100> * self.level) + "<STR_LIT:%>"<EOL><DEDENT>else:<EOL><INDENT>level_str = self.level<EOL><DEDENT>return level_str<EOL>


<EOL>if config.get("<STR_LIT>"):<EOL><INDENT>if config["<STR_LIT>"].get("<STR_LIT>"):<EOL><INDENT>self.expvar = config["<STR_LIT>"]["<STR_LIT>"].get("<STR_LIT>")<EOL>if config["<STR_LIT>"]["<STR_LIT>"].get("<STR_LIT:port>"):<EOL><INDENT>self.expvar_port = config["<STR_LIT>"]["<STR_LIT>"].get("<STR_LIT:port>")<EOL><DEDENT>else:<EOL><INDENT>self.expvar_port = self.DEFAULT_EXPVAR_PORT<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>config["<STR_LIT>"] = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": True,<EOL>}<EOL>}<EOL>self.expvar = True<EOL>self.expvar_port = self.DEFAULT_EXPVAR_PORT<EOL><DEDENT>for pool in config['<STR_LIT>']:<EOL><INDENT>if pool.get('<STR_LIT>', {}).get('<STR_LIT:file>', '<STR_LIT>'):<EOL><INDENT>self.ammofile = pool['<STR_LIT>']['<STR_LIT:file>']<EOL>pool['<STR_LIT>']['<STR_LIT:file>'] = resource_manager.resource_filename(<EOL>self.ammofile<EOL>)<EOL><DEDENT>if not pool.get('<STR_LIT:result>') or '<STR_LIT>' not in pool.get('<STR_LIT:result>', {}).get('<STR_LIT:type>', '<STR_LIT>'):<EOL><INDENT>logger.warning('<STR_LIT>')<EOL>pool['<STR_LIT:result>'] = dict(<EOL>destination=self.DEFAULT_REPORT_FILE,<EOL>type='<STR_LIT>',<EOL>)<EOL><DEDENT><DEDENT>return config<EOL>

download remote resources, replace links with local filenames
add result file section
:param dict config: pandora config


def on_aggregated_data(self, data, stats):

self.data_and_stats_stream.write(<EOL>'<STR_LIT>' % json.dumps({<EOL>'<STR_LIT:data>': data,<EOL>'<STR_LIT>': stats<EOL>}))<EOL>

@data: aggregated data
@stats: stats about gun


def get_uploader(data_session, column_mapping, overall_only=False):

overall = {col_name: data_session.new_aggregated_metric(name + '<STR_LIT>')<EOL>for col_name, name in column_mapping.items()}<EOL>def upload_df(df):<EOL><INDENT>for col_name, metric in overall.items():<EOL><INDENT>df['<STR_LIT:value>'] = df[col_name]<EOL>metric.put(df)<EOL><DEDENT><DEDENT>return upload_df<EOL>

:type column_mapping: dict
:type data_session: DataSession


def getconfig(self, filename, target_hint):

try:<EOL><INDENT>tree = self.parse_xml(filename)<EOL><DEDENT>except IOError as exc:<EOL><INDENT>logger.error("<STR_LIT>", exc)<EOL>raise RuntimeError("<STR_LIT>" % filename)<EOL><DEDENT>hosts = tree.findall('<STR_LIT>')<EOL>config = []<EOL>for host in hosts:<EOL><INDENT>host_config = self.get_host_config(host, target_hint)<EOL>config.append(host_config)<EOL><DEDENT>return config<EOL>


cfg_path = "<STR_LIT>".format(self.host)<EOL>if os.path.isfile(cfg_path):<EOL><INDENT>logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self.host)<EOL>handle, cfg_path = tempfile.mkstemp('<STR_LIT>', '<STR_LIT>')<EOL>os.close(handle)<EOL><DEDENT>try:<EOL><INDENT>config = ConfigParser.RawConfigParser()<EOL>config.add_section('<STR_LIT>')<EOL>[<EOL>config.set('<STR_LIT>', "<STR_LIT>" % idx, cmd)<EOL>for idx, cmd in enumerate(self.startups)<EOL>]<EOL>config.add_section('<STR_LIT>')<EOL>[<EOL>config.set('<STR_LIT>', "<STR_LIT>" % idx, cmd)<EOL>for idx, cmd in enumerate(self.shutdowns)<EOL>]<EOL>config.add_section('<STR_LIT:source>')<EOL>[<EOL>config.set('<STR_LIT:source>', "<STR_LIT>" % idx, path)<EOL>for idx, path in enumerate(self.sources)<EOL>]<EOL>with open(cfg_path, '<STR_LIT:w>') as fds:<EOL><INDENT>config.write(fds)<EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>',<EOL>exc,<EOL>exc_info=True)<EOL><DEDENT>return cfg_path<EOL>

Startup and shutdown commands config
        Used by agent.py on the target


cfg_path = "<STR_LIT>".format(self.host)<EOL>if os.path.isfile(cfg_path):<EOL><INDENT>logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self.host)<EOL>handle, cfg_path = tempfile.mkstemp('<STR_LIT>', '<STR_LIT>')<EOL>os.close(handle)<EOL><DEDENT>cmds = "<STR_LIT>"<EOL>for idx, cmd in enumerate(self.custom):<EOL><INDENT>cmds += "<STR_LIT>".format(idx=idx, cmd=cmd['<STR_LIT>'])<EOL><DEDENT>customs_script = """<STR_LIT>""".format(cmds=cmds)<EOL>with open(cfg_path, '<STR_LIT:w>') as fds:<EOL><INDENT>fds.write(customs_script)<EOL><DEDENT>return cfg_path<EOL>

bash script w/ custom commands inside
        inspired by half a night trying to avoid escaping bash special characters


cfg_path = "<STR_LIT>".format(self.host)<EOL>if os.path.isfile(cfg_path):<EOL><INDENT>logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self.host)<EOL>handle, cfg_path = tempfile.mkstemp('<STR_LIT>', '<STR_LIT>')<EOL>os.close(handle)<EOL><DEDENT>self.monitoring_data_output = "<STR_LIT>".format(<EOL>remote_folder=workdir)<EOL>defaults_old_enabled = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>try:<EOL><INDENT>config = ConfigParser.RawConfigParser()<EOL>config.add_section("<STR_LIT>")<EOL>config.add_section("<STR_LIT>")<EOL>config.set(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>".format(interval=self.interval))<EOL>config.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT:true>")<EOL>config.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>config.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>config.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>for section in self.host_config.keys():<EOL><INDENT>if not self.old_style_configs:<EOL><INDENT>config.add_section(<EOL>"<STR_LIT>".format(<EOL>section_name=self.host_config[section]['<STR_LIT:name>']))<EOL>for key, value in iteritems(self.host_config[section]):<EOL><INDENT>if key != '<STR_LIT:name>':<EOL><INDENT>config.set(<EOL>"<STR_LIT>".format(<EOL>section_name=self.host_config[section][<EOL>'<STR_LIT:name>']),<EOL>"<STR_LIT>".format(key=key),<EOL>"<STR_LIT>".format(value=value))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if section in defaults_old_enabled:<EOL><INDENT>config.add_section(<EOL>"<STR_LIT>".format(<EOL>section_name=self.host_config[section]['<STR_LIT:name>']))<EOL>for key, value in iteritems(self.host_config[section]):<EOL><INDENT>if key in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'<EOL>]:<EOL><INDENT>config.set(<EOL>"<STR_LIT>".format(<EOL>section_name=self.host_config[section][<EOL>'<STR_LIT:name>']),<EOL>"<STR_LIT>".format(key=key),<EOL>"<STR_LIT>".format(value=value))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>config.add_section("<STR_LIT>")<EOL>config.set(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>".format(config=self.monitoring_data_output))<EOL>config.set("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>with open(cfg_path, '<STR_LIT:w>') as fds:<EOL><INDENT>config.write(fds)<EOL><DEDENT>inputs = "<STR_LIT>"<EOL>for idx, cmd in enumerate(self.custom):<EOL><INDENT>inputs += "<STR_LIT>"<EOL>inputs += "<STR_LIT>".format(<EOL>workdir=workdir, idx=idx)<EOL>inputs += "<STR_LIT>"<EOL>inputs += "<STR_LIT>"<EOL>inputs += "<STR_LIT>".format(cmd.get('<STR_LIT:label>'))<EOL>if cmd['<STR_LIT>']:<EOL><INDENT>decoder.diff_metrics['<STR_LIT>'].append(<EOL>decoder.find_common_names(cmd.get('<STR_LIT:label>')))<EOL><DEDENT><DEDENT>with open(cfg_path, '<STR_LIT:a>') as fds:<EOL><INDENT>fds.write(inputs)<EOL><DEDENT>telegraf_raw = "<STR_LIT>"<EOL>for element in self.telegrafraw:<EOL><INDENT>telegraf_raw += element<EOL><DEDENT>with open(cfg_path, '<STR_LIT:a>') as fds:<EOL><INDENT>fds.write(telegraf_raw)<EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>',<EOL>exc,<EOL>exc_info=True)<EOL><DEDENT>return cfg_path<EOL>

Telegraf collector config,
        toml format


<EOL>agent_configs = []<EOL>if self.config:<EOL><INDENT>agent_configs = self.config_manager.getconfig(<EOL>self.config, self.default_target)<EOL><DEDENT>for config in agent_configs:<EOL><INDENT>if config['<STR_LIT:host>'] in ['<STR_LIT:localhost>', '<STR_LIT:127.0.0.1>', '<STR_LIT>']:<EOL><INDENT>client = self.clients['<STR_LIT:localhost>'](<EOL>config, self.old_style_configs, kill_old=self.kill_old)<EOL><DEDENT>else:<EOL><INDENT>client = self.clients['<STR_LIT>'](<EOL>config, self.old_style_configs, timeout=<NUM_LIT:5>, kill_old=self.kill_old)<EOL><DEDENT>logger.debug('<STR_LIT>', client.host)<EOL>agent_config, startup_config, customs_script = client.install()<EOL>if agent_config:<EOL><INDENT>self.agents.append(client)<EOL>self.artifact_files.append(agent_config)<EOL><DEDENT>if startup_config:<EOL><INDENT>self.artifact_files.append(startup_config)<EOL><DEDENT>if customs_script:<EOL><INDENT>self.artifact_files.append(customs_script)<EOL><DEDENT><DEDENT>

Prepare for monitoring - install agents etc


[agent.start() for agent in self.agents]<EOL>[agent.reader_thread.start() for agent in self.agents]<EOL>

Start agents

        execute popen of agent.py on target and start output reader thread.


start_time = time.time()<EOL>for agent in self.agents:<EOL><INDENT>for collect in agent.reader:<EOL><INDENT>if not collect:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>for chunk in collect:<EOL><INDENT>ts, prepared_results = chunk<EOL>if self.load_start_time and int(<EOL>ts) >= self.load_start_time:<EOL><INDENT>ready_to_send = {<EOL>"<STR_LIT>": int(ts),<EOL>"<STR_LIT:data>": {<EOL>self.hash_hostname(agent.host): {<EOL>"<STR_LIT>": agent.config.comment,<EOL>"<STR_LIT>": prepared_results<EOL>}<EOL>}<EOL>}<EOL>self.__collected_data.append(ready_to_send)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>logger.debug(<EOL>'<STR_LIT>',<EOL>(time.time() - start_time) * <NUM_LIT:1000>)<EOL>collected_data_length = len(self.__collected_data)<EOL>if not self.first_data_received and self.__collected_data:<EOL><INDENT>self.first_data_received = True<EOL>logger.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>self.send_collected_data()<EOL><DEDENT>return collected_data_length<EOL>


logger.debug("<STR_LIT>")<EOL>for agent in self.agents:<EOL><INDENT>log_filename, data_filename = agent.uninstall()<EOL>self.artifact_files.append(log_filename)<EOL>self.artifact_files.append(data_filename)<EOL><DEDENT>for agent in self.agents:<EOL><INDENT>try:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>', agent)<EOL>agent.reader_thread.join(<NUM_LIT:10>)<EOL><DEDENT>except BaseException:<EOL><INDENT>logger.error('<STR_LIT>', exc_info=True)<EOL><DEDENT><DEDENT>


data = self.__collected_data<EOL>self.__collected_data = []<EOL>for listener in self.listeners:<EOL><INDENT>listener.monitoring_data(copy.deepcopy(data))<EOL><DEDENT>


logger.warning("<STR_LIT>", sig)<EOL>raise KeyboardInterrupt()<EOL>

required for non-tty python runs to interrupt


logger.info('<STR_LIT>')<EOL>args = self.python.split() + [<EOL>os.path.join(<EOL>self.workdir,<EOL>self.AGENT_FILENAME),<EOL>'<STR_LIT>',<EOL>self.path['<STR_LIT>'],<EOL>'<STR_LIT>',<EOL>self.host]<EOL>if self.kill_old:<EOL><INDENT>args.append(self.kill_old)<EOL><DEDENT>self.session = self.popen(args)<EOL>self.reader_thread = threading.Thread(target=self.read_buffer)<EOL>self.reader_thread.setDaemon(True)<EOL>return self.session<EOL>


if self.session:<EOL><INDENT>logger.info('<STR_LIT>')<EOL>self.session.terminate()<EOL>self.session.wait()<EOL>self.session = None<EOL><DEDENT>log_filename = "<STR_LIT>".format(host="<STR_LIT:localhost>")<EOL>data_filename = "<STR_LIT>".format(host="<STR_LIT:localhost>")<EOL>try:<EOL><INDENT>logger.info('<STR_LIT>')<EOL>copyfile(self.workdir + "<STR_LIT>", log_filename)<EOL>copyfile(self.workdir + "<STR_LIT>", data_filename)<EOL>logger.info('<STR_LIT>', self.workdir)<EOL>rmtree(self.workdir)<EOL><DEDENT>except Exception:<EOL><INDENT>logger.error("<STR_LIT>", exc_info=True)<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>return log_filename, data_filename<EOL>


logger.info(<EOL>"<STR_LIT>",<EOL>self.username,<EOL>self.host)<EOL>cmd = self.python + '<STR_LIT>'<EOL>logger.info("<STR_LIT>", self.host)<EOL>try:<EOL><INDENT>out, errors, err_code = self.ssh.execute(cmd)<EOL><DEDENT>except Exception:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>",<EOL>self.host,<EOL>exc_info=True)<EOL>return None, None, None<EOL><DEDENT>if errors:<EOL><INDENT>logger.error("<STR_LIT>", self.host, errors)<EOL>logger.error("<STR_LIT>", self.host)<EOL>return None, None, None<EOL><DEDENT>if err_code:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>" %<EOL>(self.username, self.host, err_code, out.strip()))<EOL>return None, None, None<EOL><DEDENT>remote_dir = out.strip()<EOL>if remote_dir:<EOL><INDENT>self.path['<STR_LIT>'] = remote_dir<EOL>self.agent_remote_folder = remote_dir<EOL><DEDENT>logger.debug(<EOL>"<STR_LIT>", self.host, self.path['<STR_LIT>'])<EOL>agent_config = self.config.create_collector_config(<EOL>self.path['<STR_LIT>'])<EOL>startup_config = self.config.create_startup_config()<EOL>customs_script = self.config.create_custom_exec_script()<EOL>remote_cmd = '<STR_LIT>' + self.path[<EOL>'<STR_LIT>'] + '<STR_LIT>'<EOL>cmd = self.python + '<STR_LIT>'.format(cmd=remote_cmd)<EOL>remote_telegraf_exists = "<STR_LIT:False>"<EOL>try:<EOL><INDENT>out, err, err_code = self.ssh.execute(cmd)<EOL><DEDENT>except Exception:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>",<EOL>self.host,<EOL>exc_info=True)<EOL><DEDENT>else:<EOL><INDENT>if err:<EOL><INDENT>logger.error("<STR_LIT>", self.host, errors)<EOL><DEDENT>if out.strip():<EOL><INDENT>remote_telegraf_exists = out.strip()<EOL><DEDENT><DEDENT>try:<EOL><INDENT>if remote_telegraf_exists in "<STR_LIT:True>":<EOL><INDENT>logger.debug('<STR_LIT>', self.host)<EOL><DEDENT>else:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>',<EOL>self.host)<EOL>if os.path.isfile(self.path['<STR_LIT>']):<EOL><INDENT>self.ssh.send_file(<EOL>self.path['<STR_LIT>'],<EOL>self.path['<STR_LIT>'])<EOL><DEDENT>elif os.path.isfile("<STR_LIT>"):<EOL><INDENT>self.ssh.send_file(<EOL>'<STR_LIT>', self.path['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', self.host, self.path['<STR_LIT>'])<EOL>return None, None, None<EOL><DEDENT><DEDENT>self.ssh.send_file(<EOL>os.path.join(<EOL>self.path['<STR_LIT>'],<EOL>self.AGENT_FILENAME),<EOL>os.path.join(<EOL>self.path['<STR_LIT>'],<EOL>self.AGENT_FILENAME))<EOL>self.ssh.send_file(<EOL>agent_config,<EOL>os.path.join(<EOL>self.path['<STR_LIT>'],<EOL>'<STR_LIT>'))<EOL>self.ssh.send_file(<EOL>startup_config,<EOL>os.path.join(<EOL>self.path['<STR_LIT>'],<EOL>'<STR_LIT>'))<EOL>self.ssh.send_file(<EOL>customs_script,<EOL>os.path.join(<EOL>self.path['<STR_LIT>'],<EOL>'<STR_LIT>'))<EOL><DEDENT>except Exception:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>", self.host, exc_info=True)<EOL>return None, None, None<EOL><DEDENT>return agent_config, startup_config, customs_script<EOL>

Create folder and copy agent and metrics scripts to remote host


logger.info('<STR_LIT>', self.host)<EOL>command = "<STR_LIT>".format(<EOL>python=self.python,<EOL>agent_path=os.path.join(<EOL>self.path['<STR_LIT>'],<EOL>self.AGENT_FILENAME),<EOL>telegraf_path=self.path['<STR_LIT>'],<EOL>host=self.host,<EOL>kill_old=self.kill_old)<EOL>logger.debug('<STR_LIT>', command)<EOL>self.session = self.ssh.async_session(command)<EOL>self.reader_thread = threading.Thread(target=self.read_buffer)<EOL>self.reader_thread.setDaemon(True)<EOL>return self.session<EOL>


collect = []<EOL>if block:<EOL><INDENT>for chunk in block.split('<STR_LIT:\n>'):<EOL><INDENT>try:<EOL><INDENT>if chunk:<EOL><INDENT>prepared_results = {}<EOL>jsn = json.loads(chunk)<EOL>for ts, values in jsn.iteritems():<EOL><INDENT>for key, value in values.iteritems():<EOL><INDENT>try:<EOL><INDENT>key_group, key_name = key.split('<STR_LIT:_>')[<NUM_LIT:0>].split('<STR_LIT:->')[<NUM_LIT:0>], '<STR_LIT:_>'.join(key.split('<STR_LIT:_>')[<NUM_LIT:1>:])<EOL><DEDENT>except:  <EOL><INDENT>key_group, key_name = key.split('<STR_LIT:_>')[<NUM_LIT:0>], '<STR_LIT:_>'.join(key.split('<STR_LIT:_>')[<NUM_LIT:1>:])<EOL><DEDENT>if key_group in decoder.diff_metrics.keys():<EOL><INDENT>if key_name in decoder.diff_metrics[key_group]:<EOL><INDENT>decoded_key = decoder.find_common_names(<EOL>key)<EOL>if self.prev_check:<EOL><INDENT>try:<EOL><INDENT>value = jsn[ts][key] -self.prev_check[key]<EOL><DEDENT>except KeyError:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', key, ts, exc_info=True)<EOL>value = <NUM_LIT:0><EOL><DEDENT>prepared_results[decoded_key] = value<EOL><DEDENT><DEDENT>else:<EOL><INDENT>decoded_key = decoder.find_common_names(<EOL>key)<EOL>prepared_results[decoded_key] = value<EOL><DEDENT><DEDENT>else:<EOL><INDENT>decoded_key = decoder.find_common_names(<EOL>key)<EOL>prepared_results[decoded_key] = value<EOL><DEDENT><DEDENT>self.prev_check = jsn[ts]<EOL>collect.append((ts, prepared_results))<EOL><DEDENT><DEDENT><DEDENT>except ValueError:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>', chunk)<EOL>logger.debug(<EOL>'<STR_LIT>',<EOL>exc_info=True)<EOL>return []<EOL><DEDENT>except BaseException:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>',<EOL>chunk,<EOL>exc_info=True)<EOL>return []<EOL><DEDENT><DEDENT>if collect:<EOL><INDENT>return collect<EOL><DEDENT><DEDENT>


self.known_metrics = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>}<EOL>self.diff_metrics = {<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>'<STR_LIT>': ['<STR_LIT>'],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>'<STR_LIT>': [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'<EOL>],<EOL>'<STR_LIT>': []<EOL>}<EOL>

translates telegraf metric names into common Monitoring metric names
translates `uncommon` names to `custom:%s`s


try:<EOL><INDENT>is_telegraf = self.core.get_option('<STR_LIT>', "<STR_LIT>")<EOL><DEDENT>except KeyError:<EOL><INDENT>is_telegraf = None<EOL><DEDENT>try:<EOL><INDENT>is_monitoring = self.core.get_option('<STR_LIT>', "<STR_LIT>")<EOL><DEDENT>except KeyError:<EOL><INDENT>is_monitoring = None<EOL><DEDENT>if is_telegraf and is_monitoring:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if is_telegraf and not is_monitoring:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if not is_telegraf and is_monitoring:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if not is_telegraf and not is_monitoring:<EOL><INDENT>try:<EOL><INDENT>is_telegraf_dt = self.core.get_option('<STR_LIT>')<EOL><DEDENT>except NoOptionError:<EOL><INDENT>is_telegraf_dt = None<EOL><DEDENT>try:<EOL><INDENT>is_monitoring_dt = self.core.get_option('<STR_LIT>')<EOL><DEDENT>except BaseException:<EOL><INDENT>is_monitoring_dt = None<EOL><DEDENT>if is_telegraf_dt and is_monitoring_dt:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if is_telegraf_dt and not is_monitoring_dt:<EOL><INDENT>return<EOL><DEDENT>if not is_telegraf_dt and is_monitoring_dt:<EOL><INDENT>self.core.set_option(<EOL>"<STR_LIT>", "<STR_LIT>", is_monitoring_dt)<EOL><DEDENT>if not is_telegraf_dt and not is_monitoring_dt:<EOL><INDENT>return<EOL><DEDENT><DEDENT>

we need to be flexible in order to determine which plugin's configuration
specified and make appropriate configs to metrics collector

:return: SECTION name or None for defaults


if self._config is None:<EOL><INDENT>value = self.get_option('<STR_LIT>')<EOL>if value.lower() == "<STR_LIT:none>":<EOL><INDENT>self.monitoring = None<EOL>self.die_on_fail = False<EOL>self._config = value<EOL><DEDENT>else:<EOL><INDENT>if value.startswith("<STR_LIT:<>"):<EOL><INDENT>config_contents = value<EOL><DEDENT>elif value.lower() == "<STR_LIT>":<EOL><INDENT>self.die_on_fail = False<EOL>with open(resource.resource_filename(self.default_config), '<STR_LIT:rb>') as def_config:<EOL><INDENT>config_contents = def_config.read()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>with open(resource.resource_filename(value), '<STR_LIT:rb>') as config:<EOL><INDENT>config_contents = config.read()<EOL><DEDENT><DEDENT>self._config = self._save_config_contents(config_contents)<EOL><DEDENT><DEDENT>return self._config<EOL>


logger.debug("<STR_LIT>")<EOL>if self.store:<EOL><INDENT>self.store.close()<EOL><DEDENT>


def __handle_data_items(self, host, data):

for metric, value in data.iteritems():<EOL><INDENT>if value == '<STR_LIT>':<EOL><INDENT>self.sign[host][metric] = -<NUM_LIT:1><EOL>self.data[host][metric] = value<EOL><DEDENT>else:<EOL><INDENT>if not self.data[host].get(metric, None):<EOL><INDENT>self.sign[host][metric] = <NUM_LIT:1><EOL><DEDENT>elif float(value) > float(self.data[host][metric]):<EOL><INDENT>self.sign[host][metric] = <NUM_LIT:1><EOL><DEDENT>elif float(value) < float(self.data[host][metric]):<EOL><INDENT>self.sign[host][metric] = -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self.sign[host][metric] = <NUM_LIT:0><EOL><DEDENT>self.data[host][metric] = "<STR_LIT>" % float(value)<EOL><DEDENT><DEDENT>

store metric in data tree and calc offset signs

        sign < 0 is CYAN, means metric value is lower then previous,
        sign > 1 is YELLOW, means metric value is higher then prevoius,
        sign == 0 is WHITE, means initial or equal metric value


raise NotImplementedError()<EOL>


points = list()<EOL>for second_data in data:<EOL><INDENT>for host, host_data in second_data["<STR_LIT:data>"].iteritems():<EOL><INDENT>points.append(<EOL>self.__make_points(<EOL>"<STR_LIT>",<EOL>{"<STR_LIT:host>": host, "<STR_LIT>": host_data.get("<STR_LIT>")},<EOL>second_data["<STR_LIT>"],<EOL>{<EOL>metric: value<EOL>for metric, value in host_data["<STR_LIT>"].iteritems()<EOL>}<EOL>)<EOL>)<EOL><DEDENT><DEDENT>return points<EOL>

The reason why we have two separate methods for monitoring
and aggregates is a strong difference in incoming data.


def __make_points_for_label(self, ts, data, label, prefix, gun_stats):

label_points = list()<EOL>label_points.extend(<EOL>(<EOL>self.__make_points(<EOL>prefix + "<STR_LIT>",<EOL>{"<STR_LIT:label>": label},<EOL>ts,<EOL>self.__make_quantile_fields(data)<EOL>),<EOL>self.__make_points(<EOL>prefix + "<STR_LIT>",<EOL>{"<STR_LIT:label>": label},<EOL>ts,<EOL>self.__make_overall_meta_fields(data, gun_stats)<EOL>),<EOL>self.__make_points(<EOL>prefix + "<STR_LIT>",<EOL>{"<STR_LIT:label>": label},<EOL>ts,<EOL>self.__make_netcodes_fields(data)<EOL>),<EOL>self.__make_points(<EOL>prefix + "<STR_LIT>",<EOL>{"<STR_LIT:label>": label},<EOL>ts,<EOL>self.__make_protocodes_fields(data)<EOL>)<EOL>)<EOL>)<EOL>if self.histograms:<EOL><INDENT>for bin_, count in zip(data["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"],<EOL>data["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT:data>"]):<EOL><INDENT>label_points.append(<EOL>self.__make_points(<EOL>prefix + "<STR_LIT>",<EOL>{"<STR_LIT:label>": label},<EOL>ts,<EOL>{"<STR_LIT>": bin_, "<STR_LIT:count>": count}<EOL>)<EOL>)<EOL><DEDENT><DEDENT>return label_points<EOL>

x
        Make a set of points for `this` label

        overall_quantiles, overall_meta, net_codes, proto_codes, histograms


def __make_points(self, measurement, additional_tags, ts, fields):

tags = self.tags.copy()<EOL>tags.update(additional_tags)<EOL>return {<EOL>"<STR_LIT>": measurement,<EOL>"<STR_LIT>": tags,<EOL>"<STR_LIT:time>": int(ts),<EOL>"<STR_LIT>": fields,<EOL>}<EOL>

Parameters
----------
measurement : string
    measurement type (e.g. monitoring, overall_meta, net_codes, proto_codes, overall_quantiles)
additional_tags : dict
    custom additional tags for this points
ts : integer
    timestamp
fields : dict
    influxdb columns

Returns
-------
dict
    points for InfluxDB client


request = requests.Request(<EOL>http_method,<EOL>self.writer_url,<EOL>params=params,<EOL>json=json,<EOL>headers={<EOL>'<STR_LIT>': self.user_agent})<EOL>ids = id_gen(str(uuid.uuid4()))<EOL>network_timeouts = self.network_timeouts()<EOL>maintenance_timeouts = self.maintenance_timeouts()<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>response = self.__send_single_request(request, ids.next(), trace=trace)<EOL>return response<EOL><DEDENT>except (Timeout, ConnectionError, ProtocolError):<EOL><INDENT>logger.warn(traceback.format_exc())<EOL>try:<EOL><INDENT>timeout = next(network_timeouts)<EOL>logger.warn(<EOL>"<STR_LIT>" %<EOL>timeout)<EOL>time.sleep(timeout)<EOL>continue<EOL><DEDENT>except StopIteration:<EOL><INDENT>raise self.NetworkError()<EOL><DEDENT><DEDENT>except self.UnderMaintenance as e:<EOL><INDENT>try:<EOL><INDENT>timeout = next(maintenance_timeouts)<EOL>logger.warn(<EOL>"<STR_LIT>" %<EOL>timeout)<EOL>time.sleep(timeout)<EOL>continue<EOL><DEDENT>except StopIteration:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT><DEDENT>


if not notify_list:<EOL><INDENT>notify_list = []<EOL><DEDENT>data = {<EOL>'<STR_LIT>': task,<EOL>'<STR_LIT>': person,<EOL>'<STR_LIT>': tank,<EOL>'<STR_LIT:host>': target_host,<EOL>'<STR_LIT:port>': target_port,<EOL>'<STR_LIT>': loadscheme,<EOL>'<STR_LIT>': detailed_time,<EOL>'<STR_LIT>': notify_list<EOL>}<EOL>logger.debug("<STR_LIT>", data)<EOL>api_timeouts = self.api_timeouts()<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>response = self.__post(<EOL>"<STR_LIT>", data, trace=trace)[<NUM_LIT:0>]<EOL>return response['<STR_LIT>'], response['<STR_LIT>']<EOL><DEDENT>except (self.NotAvailable, self.StoppedFromOnline) as e:<EOL><INDENT>try:<EOL><INDENT>timeout = next(api_timeouts)<EOL>logger.warn("<STR_LIT>" % timeout)<EOL>time.sleep(timeout)<EOL>continue<EOL><DEDENT>except StopIteration:<EOL><INDENT>logger.warn('<STR_LIT>')<EOL>raise self.JobNotCreated(e.message)<EOL><DEDENT><DEDENT>except requests.HTTPError as e:<EOL><INDENT>raise self.JobNotCreated('<STR_LIT>'.format(e.response.content))<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.warn('<STR_LIT>')<EOL>logger.warn(repr(e), )<EOL>raise self.JobNotCreated()<EOL><DEDENT><DEDENT>

:return: job_nr, upload_token
:rtype: tuple


def second_data_to_push_item(self, data, stat, timestamp, overall, case):

api_data = {<EOL>'<STR_LIT>': overall,<EOL>'<STR_LIT>': case,<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:time>': str(timestamp),<EOL>'<STR_LIT>': stat["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': data["<STR_LIT>"]["<STR_LIT>"] / <NUM_LIT> / data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>':<EOL><NUM_LIT:0>,  <EOL>'<STR_LIT:input>': data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': data["<STR_LIT>"]["<STR_LIT>"] / <NUM_LIT> / data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>':<EOL>data["<STR_LIT>"]["<STR_LIT>"] / <NUM_LIT> / data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>':<EOL>data["<STR_LIT>"]["<STR_LIT>"] / <NUM_LIT> / data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': data["<STR_LIT>"]["<STR_LIT>"] / <NUM_LIT> / data["<STR_LIT>"]["<STR_LIT>"],<EOL>'<STR_LIT>': stat["<STR_LIT>"]["<STR_LIT>"],  <EOL>}<EOL>}<EOL>for q, value in zip(data["<STR_LIT>"]["<STR_LIT:q>"]["<STR_LIT:q>"],<EOL>data["<STR_LIT>"]["<STR_LIT:q>"]["<STR_LIT:value>"]):<EOL><INDENT>api_data['<STR_LIT>']['<STR_LIT:q>' + str(q)] = value / <NUM_LIT><EOL><DEDENT>for code, cnt in data["<STR_LIT>"]["<STR_LIT:count>"].items():<EOL><INDENT>api_data['<STR_LIT>'].append({'<STR_LIT:code>': int(code),<EOL>'<STR_LIT:count>': int(cnt)})<EOL><DEDENT>for code, cnt in data["<STR_LIT>"]["<STR_LIT:count>"].items():<EOL><INDENT>api_data['<STR_LIT>'].append({'<STR_LIT:code>': int(code),<EOL>'<STR_LIT:count>': int(cnt)})<EOL><DEDENT>api_data['<STR_LIT>'] = self.convert_hist(data["<STR_LIT>"][<EOL>"<STR_LIT>"])<EOL>return api_data<EOL>


if not self.lp_job.interrupted.is_set():<EOL><INDENT>self.data_queue.put((data, stats))<EOL><DEDENT>


if self._lp_job is None:<EOL><INDENT>self._lp_job = self.__get_lp_job()<EOL>self.core.publish(self.SECTION, '<STR_LIT>', self._lp_job.number)<EOL>self.core.publish(self.SECTION, '<STR_LIT>', self._lp_job.web_link)<EOL>self.core.publish(self.SECTION, '<STR_LIT>', self._lp_job.name)<EOL>self.core.publish(self.SECTION, '<STR_LIT>', self._lp_job.description)<EOL>self.core.publish(self.SECTION, '<STR_LIT>', self._lp_job.person)<EOL>self.core.publish(self.SECTION, '<STR_LIT>', self._lp_job.task)<EOL>self.core.publish(self.SECTION, '<STR_LIT:version>', self._lp_job.version)<EOL><DEDENT>return self._lp_job<EOL>


api_client = self.__get_api_client()<EOL>info = self.get_generator_info()<EOL>port = info.port<EOL>loadscheme = [] if isinstance(info.rps_schedule, (str, dict)) else info.rps_schedule<EOL>lp_job = LPJob(client=api_client,<EOL>target_host=self.target,<EOL>target_port=port,<EOL>number=self.cfg.get('<STR_LIT>', None),<EOL>token=self.get_option('<STR_LIT>'),<EOL>person=self.__get_operator(),<EOL>task=self.task,<EOL>name=self.get_option('<STR_LIT>', '<STR_LIT:none>'),<EOL>description=self.get_option('<STR_LIT>'),<EOL>tank=self.core.job.tank,<EOL>notify_list=self.get_option("<STR_LIT>"),<EOL>load_scheme=loadscheme,<EOL>version=self.get_option('<STR_LIT>'),<EOL>log_data_requests=self.get_option('<STR_LIT>'),<EOL>log_monitoring_requests=self.get_option('<STR_LIT>'),<EOL>log_status_requests=self.get_option('<STR_LIT>'),<EOL>log_other_requests=self.get_option('<STR_LIT>'),<EOL>add_cleanup=lambda: self.add_cleanup(self.close_job))<EOL>lp_job.send_config(LPRequisites.CONFIGINITIAL, yaml.dump(self.core.configinitial))<EOL>return lp_job<EOL>


assert bool(number) == bool(<EOL>token), '<STR_LIT>'<EOL>self.log_other_requests = log_other_requests<EOL>self.log_data_requests = log_data_requests<EOL>self.log_status_requests = log_status_requests<EOL>self.log_monitoring_requests = log_monitoring_requests<EOL>self.name = name<EOL>self.tank = tank<EOL>self.target_host = target_host<EOL>self.target_port = target_port<EOL>self.person = person<EOL>self.task = task<EOL>self.interrupted = threading.Event()<EOL>self._number = number<EOL>self._token = token<EOL>self.api_client = client<EOL>self.notify_list = notify_list<EOL>self.description = description<EOL>self.version = version<EOL>self.detailed_time = detailed_time<EOL>self.load_scheme = load_scheme<EOL>self.is_finished = False<EOL>self.web_link = '<STR_LIT>'<EOL>self.add_cleanup = add_cleanup<EOL>

:param client: APIClient
:param log_data_requests: bool
:param log_other_request: bool
:param log_status_requests: bool
:param log_monitoring_requests: bool


if self.__stderr_file:<EOL><INDENT>self.__stderr_file.close()<EOL><DEDENT>if not self.__process:<EOL><INDENT>return<EOL><DEDENT>waitfor = time.time() + _PROCESS_KILL_TIMEOUT<EOL>while time.time() < waitfor:<EOL><INDENT>try:<EOL><INDENT>self.__process.terminate()<EOL><DEDENT>except EnvironmentError as e:<EOL><INDENT>if e.errno != errno.ESRCH:<EOL><INDENT>_LOGGER.warning("<STR_LIT>".format(self.__cmd, e))<EOL><DEDENT>return<EOL><DEDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT>try:<EOL><INDENT>self.__process.kill()<EOL><DEDENT>except EnvironmentError as e:<EOL><INDENT>if e.errno != errno.ESRCH:<EOL><INDENT>_LOGGER.warning("<STR_LIT>".format(self.__cmd, e))<EOL><DEDENT>return<EOL><DEDENT>

Gracefull termination of running process


results = []<EOL>for line in lines:<EOL><INDENT>timestamp, rps, instances = line.split("<STR_LIT:\t>")<EOL>curr_ts = int(float(timestamp))  <EOL>if self.__last_ts < curr_ts:<EOL><INDENT>self.__last_ts = curr_ts<EOL>results.append(self.stats_item(self.__last_ts, float(rps), float(instances)))<EOL><DEDENT><DEDENT>return results<EOL>


default_size = (<NUM_LIT:30>, <NUM_LIT>)<EOL>env = os.environ<EOL>def ioctl_gwinsz(file_d):<EOL><INDENT>'''<STR_LIT>'''<EOL>try:<EOL><INDENT>sizes = struct.unpack(<EOL>'<STR_LIT>', fcntl.ioctl(file_d, termios.TIOCGWINSZ, '<STR_LIT>'))<EOL><DEDENT>except Exception:<EOL><INDENT>sizes = default_size<EOL><DEDENT>return sizes<EOL><DEDENT>sizes = ioctl_gwinsz(<NUM_LIT:0>) or ioctl_gwinsz(<NUM_LIT:1>) or ioctl_gwinsz(<NUM_LIT:2>)<EOL>if not sizes:<EOL><INDENT>try:<EOL><INDENT>file_d = os.open(os.ctermid(), os.O_RDONLY)<EOL>sizes = ioctl_gwinsz(file_d)<EOL>os.close(file_d.fileno())<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if not sizes:<EOL><INDENT>try:<EOL><INDENT>sizes = (env['<STR_LIT>'], env['<STR_LIT>'])<EOL><DEDENT>except Exception:<EOL><INDENT>sizes = default_size<EOL><DEDENT><DEDENT>return int(sizes[<NUM_LIT:1>]), int(sizes[<NUM_LIT:0>])<EOL>

Gets width and height of terminal viewport


right_line = '<STR_LIT>'<EOL>if widget_output:<EOL><INDENT>right_line = widget_output.pop(<NUM_LIT:0>)<EOL>if len(right_line) > self.right_panel_width:<EOL><INDENT>right_line_plain = self.markup.clean_markup(right_line)<EOL>if len(right_line_plain) > self.right_panel_width:<EOL><INDENT>right_line = right_line[:self.right_panel_width] + self.markup.RESET<EOL><DEDENT><DEDENT><DEDENT>return right_line<EOL>


def __truncate(self, line_arr, max_width):

def is_space(chunk):<EOL><INDENT>return all([True if i == '<STR_LIT:U+0020>' else False for i in chunk])<EOL><DEDENT>def is_empty(chunks, markups):<EOL><INDENT>result = []<EOL>for chunk in chunks:<EOL><INDENT>if chunk in markups:<EOL><INDENT>result.append(True)<EOL><DEDENT>elif is_space(chunk):<EOL><INDENT>result.append(True)<EOL><DEDENT>else:<EOL><INDENT>result.append(False)<EOL><DEDENT><DEDENT>return all(result)<EOL><DEDENT>left = max_width<EOL>result = '<STR_LIT>'<EOL>markups = self.markup.get_markup_vars()<EOL>for num, chunk in enumerate(line_arr):<EOL><INDENT>if chunk in markups:<EOL><INDENT>result += chunk<EOL><DEDENT>else:<EOL><INDENT>if left > <NUM_LIT:0>:<EOL><INDENT>if len(chunk) <= left:<EOL><INDENT>result += chunk<EOL>left -= len(chunk)<EOL><DEDENT>else:<EOL><INDENT>leftover = (chunk[left:],) + line_arr[num + <NUM_LIT:1>:]<EOL>was_cut = not is_empty(leftover, markups)<EOL>if was_cut:<EOL><INDENT>result += chunk[:left - <NUM_LIT:1>] + self.markup.RESET + '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>result += chunk[:left]<EOL><DEDENT>left = <NUM_LIT:0><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return result<EOL>

Cut tuple of line chunks according to it's wisible lenght


self.log.debug("<STR_LIT>")<EOL>left_block = self.left_panel<EOL>left_block.render()<EOL>blank_space = self.left_panel_width - left_block.width<EOL>lines = []<EOL>pre_space = '<STR_LIT:U+0020>' * int(blank_space / <NUM_LIT:2>)<EOL>if not left_block.lines:<EOL><INDENT>lines = [('<STR_LIT>'), (self.markup.RED + '<STR_LIT>' + self.markup.RESET)]<EOL><DEDENT>else:<EOL><INDENT>while self.left_panel.lines:<EOL><INDENT>src_line = self.left_panel.lines.pop(<NUM_LIT:0>)<EOL>line = pre_space + self.__truncate(src_line, self.left_panel_width)<EOL>post_space = '<STR_LIT:U+0020>' * (self.left_panel_width - len(self.markup.clean_markup(line)))<EOL>line += post_space + self.markup.RESET<EOL>lines.append(line)<EOL><DEDENT><DEDENT>return lines<EOL>


self.term_width, self.term_height = get_terminal_size()<EOL>self.log.debug(<EOL>"<STR_LIT>", self.term_width, self.term_height)<EOL>self.right_panel_width = int(<EOL>(self.term_width - len(self.RIGHT_PANEL_SEPARATOR))<EOL>* (float(self.info_panel_percent) / <NUM_LIT:100>)) - <NUM_LIT:1><EOL>if self.right_panel_width > <NUM_LIT:0>:<EOL><INDENT>self.left_panel_width = self.term_width -self.right_panel_width - len(self.RIGHT_PANEL_SEPARATOR) - <NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>self.right_panel_width = <NUM_LIT:0><EOL>self.left_panel_width = self.term_width - <NUM_LIT:1><EOL><DEDENT>self.log.debug(<EOL>"<STR_LIT>", self.left_panel_width,<EOL>self.right_panel_width)<EOL>widget_output = []<EOL>if self.right_panel_width:<EOL><INDENT>widget_output = []<EOL>self.log.debug("<STR_LIT>" % len(self.info_widgets))<EOL>for index, widget in sorted(<EOL>iter(self.info_widgets.items()),<EOL>key=lambda item: (item[<NUM_LIT:1>].get_index(), item[<NUM_LIT:0>])):<EOL><INDENT>self.log.debug("<STR_LIT>", index, widget)<EOL>widget_out = widget.render(self).strip()<EOL>if widget_out:<EOL><INDENT>widget_output += widget_out.split("<STR_LIT:\n>")<EOL>widget_output += ["<STR_LIT>"]<EOL><DEDENT><DEDENT><DEDENT>left_lines = self.__render_left_panel()<EOL>self.log.debug("<STR_LIT>")<EOL>output = []<EOL>for line_no in range(<NUM_LIT:1>, self.term_height):<EOL><INDENT>line = "<STR_LIT:U+0020>"<EOL>if line_no > <NUM_LIT:1> and left_lines:<EOL><INDENT>left_line = left_lines.pop(<NUM_LIT:0>)<EOL>left_line_plain = self.markup.clean_markup(left_line)<EOL>left_line += (<EOL>'<STR_LIT:U+0020>' * (self.left_panel_width - len(left_line_plain)))<EOL>line += left_line<EOL><DEDENT>else:<EOL><INDENT>line += '<STR_LIT:U+0020>' * self.left_panel_width<EOL><DEDENT>if self.right_panel_width:<EOL><INDENT>line += self.markup.RESET<EOL>line += self.markup.WHITE<EOL>line += self.RIGHT_PANEL_SEPARATOR<EOL>line += self.markup.RESET<EOL>right_line = self.__get_right_line(widget_output)<EOL>line += right_line<EOL><DEDENT>output.append(line)<EOL><DEDENT>return self.markup.new_line.join(output) + self.markup.new_line<EOL>


index = widget.get_index()<EOL>while index in list(self.info_widgets.keys()):<EOL><INDENT>index += <NUM_LIT:1><EOL><DEDENT>self.info_widgets[widget.get_index()] = widget<EOL>

Add widget string to right panel of the screen


Notification method about new aggregator data


pass<EOL>


result = []<EOL>width = max([self.clean_len(line) for line in prepared])<EOL>for line in prepared:<EOL><INDENT>spacer = '<STR_LIT:U+0020>' * (width - self.clean_len(line))<EOL>result.append(line + (self.screen.markup.RESET, spacer))<EOL><DEDENT>return (width, result)<EOL>


if isinstance(line, str):<EOL><INDENT>return len(self.screen.markup.clean_markup(line))<EOL><DEDENT>elif isinstance(line, tuple) or isinstance(line, list):<EOL><INDENT>markups = self.screen.markup.get_markup_vars()<EOL>length = <NUM_LIT:0><EOL>for i in line:<EOL><INDENT>if i not in markups:<EOL><INDENT>length += len(i)<EOL><DEDENT><DEDENT>return length<EOL><DEDENT>


raise RuntimeError("<STR_LIT>")<EOL>

Render method, fills .lines and .width properties with rendered data


if not self.screen:<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>self.screen.add_info_widget(widget)<EOL><DEDENT>


for val in self.get_markup_vars():<EOL><INDENT>orig_str = orig_str.replace(val, '<STR_LIT>')<EOL><DEDENT>return orig_str<EOL>


logger = logging.getLogger('<STR_LIT>')<EOL>self.log_filename = log_filename<EOL>self.core.add_artifact_file(self.log_filename)<EOL>file_handler = logging.FileHandler(self.log_filename)<EOL>file_handler.setLevel(logging.DEBUG)<EOL>file_handler.setFormatter(<EOL>logging.Formatter(<EOL>"<STR_LIT>"))<EOL>logger.addHandler(file_handler)<EOL>console_handler = logging.StreamHandler(sys.stdout)<EOL>stderr_hdl = logging.StreamHandler(sys.stderr)<EOL>fmt_regular = logging.Formatter(<EOL>"<STR_LIT>", "<STR_LIT>")<EOL>console_handler.setLevel(logging.INFO)<EOL>console_handler.setFormatter(fmt_regular)<EOL>stderr_hdl.setFormatter(fmt_regular)<EOL>f_err = SingleLevelFilter(logging.ERROR, True)<EOL>f_warn = SingleLevelFilter(logging.WARNING, True)<EOL>f_crit = SingleLevelFilter(logging.CRITICAL, True)<EOL>console_handler.addFilter(f_err)<EOL>console_handler.addFilter(f_warn)<EOL>console_handler.addFilter(f_crit)<EOL>logger.addHandler(console_handler)<EOL>f_info = SingleLevelFilter(logging.INFO, True)<EOL>f_debug = SingleLevelFilter(logging.DEBUG, True)<EOL>stderr_hdl.addFilter(f_info)<EOL>stderr_hdl.addFilter(f_debug)<EOL>logger.addHandler(stderr_hdl)<EOL>


if self.options.get('<STR_LIT>', None):<EOL><INDENT>self.core.apply_shorthand_options(self.options['<STR_LIT>'])<EOL><DEDENT>

override config options with user specified options


self.options = options<EOL>if self.options.get('<STR_LIT>', None):<EOL><INDENT>self.core.set_option(self.core.SECTION, "<STR_LIT>", self.options['<STR_LIT>'])<EOL><DEDENT>if self.options.get('<STR_LIT>', None):<EOL><INDENT>self.core.set_option(self.core.SECTION, '<STR_LIT>', self.options['<STR_LIT>'])<EOL><DEDENT>while True:<EOL><INDENT>try:<EOL><INDENT>self.core.get_lock()<EOL>break<EOL><DEDENT>except Exception as exc:<EOL><INDENT>if self.options.get('<STR_LIT>', None):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>self.log.info(<EOL>"<STR_LIT>",<EOL>str(exc))<EOL>time.sleep(<NUM_LIT:5>)<EOL><DEDENT><DEDENT>configs = self.get_default_configs()<EOL>if self.options.get('<STR_LIT>', None):<EOL><INDENT>configs.append(self.options['<STR_LIT>'])<EOL><DEDENT>self.core.load_configs(configs)<EOL>self.__add_user_options()<EOL>self.core.load_plugins()<EOL>if self.options.get('<STR_LIT>', None):<EOL><INDENT>self.core.set_option(self.core.SECTION, self.IGNORE_LOCKS, "<STR_LIT:1>")<EOL><DEDENT>


<EOL>configs = [resource_filename(__name__, '<STR_LIT>')]<EOL>try:<EOL><INDENT>conf_files = sorted(os.listdir(self.baseconfigs_location))<EOL>for filename in conf_files:<EOL><INDENT>if fnmatch.fnmatch(filename, '<STR_LIT>'):<EOL><INDENT>configs += [<EOL>os.path.realpath(<EOL>self.baseconfigs_location + os.sep + filename)<EOL>]<EOL><DEDENT><DEDENT><DEDENT>except OSError:<EOL><INDENT>self.log.warn(<EOL>self.baseconfigs_location + '<STR_LIT>')<EOL><DEDENT>configs += [os.path.expanduser('<STR_LIT>')]<EOL>return configs<EOL>

returns default configs list, from /etc, home dir and package_data


retcode = <NUM_LIT:1><EOL>self.log.info("<STR_LIT>")<EOL>retcode = self.core.plugins_end_test(retcode)<EOL>retcode = self.core.plugins_post_process(retcode)<EOL>self.log.info("<STR_LIT>")<EOL>return retcode<EOL>


return filter(lambda option: not is_option_deprecated(plugin, option[<NUM_LIT:0>]), options)<EOL>


defaults = cfg_ini.defaults()<EOL>options = cfg_ini.items(section) if cfg_ini.has_section(section) else []<EOL>return [(key, value) for key, value in options if key not in defaults.keys()]<EOL>

:rtype: (str, str)
:type cfg_ini: ConfigParser


return [Section(section.lower(),<EOL>guess_plugin(section.lower()),<EOL>without_defaults(cfg_ini, section))<EOL>for section in cfg_ini.sections()<EOL>if not re.match(CORE_SECTION_PATTERN, section.lower()) and section.lower() not in DEPRECATED_SECTIONS]<EOL>


DEPRECATED_PLUGINS = ['<STR_LIT>', '<STR_LIT>']<EOL>plugin_instances = [PluginInstance(key.split('<STR_LIT:_>')[<NUM_LIT:1>], value) for key, value in core_opts if<EOL>key.startswith(PLUGIN_PREFIX) and value not in DEPRECATED_PLUGINS]<EOL>enabled_instances = {instance.section_name: instance for instance in plugin_instances if instance.enabled}<EOL>disabled_instances = {instance.section_name: instance for instance in plugin_instances if not instance.enabled}<EOL>for section in sections:<EOL><INDENT>if section.name in enabled_instances.keys():<EOL><INDENT>section.enabled = True<EOL>enabled_instances.pop(section.name)<EOL><DEDENT>elif section.name in disabled_instances.keys():<EOL><INDENT>section.enabled = False<EOL>disabled_instances.pop(section.name)<EOL><DEDENT><DEDENT>for plugin_instance in [i for i in plugin_instances if<EOL>i.section_name in enabled_instances.keys() + disabled_instances.keys()]:<EOL><INDENT>sections.append(Section(plugin_instance.section_name, plugin_instance.plugin_name, [], plugin_instance.enabled))<EOL><DEDENT>return sections<EOL>


PLUGINS_TO_COMBINE = {<EOL>'<STR_LIT>': ('<STR_LIT>', '<STR_LIT>', True),<EOL>'<STR_LIT>': ('<STR_LIT>', '<STR_LIT>', False)<EOL>}<EOL>plugins = {}<EOL>ready_sections = []<EOL>for section in sections:<EOL><INDENT>if section.plugin in PLUGINS_TO_COMBINE.keys():<EOL><INDENT>try:<EOL><INDENT>plugins[section.plugin].append(section)<EOL><DEDENT>except KeyError:<EOL><INDENT>plugins[section.plugin] = [section]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ready_sections.append(section)<EOL><DEDENT><DEDENT>for plugin_name, _sections in plugins.items():<EOL><INDENT>if isinstance(_sections, list):<EOL><INDENT>parent_name, child_name, is_list = PLUGINS_TO_COMBINE[plugin_name]<EOL>ready_sections.append(Section.from_multiple(_sections, parent_name, child_name, is_list))<EOL><DEDENT><DEDENT>return ready_sections<EOL>

:type sections: list of Section
:rtype: list of Section


section_name, option_name = key.strip().split('<STR_LIT:.>', <NUM_LIT:1>)<EOL>if not re.match(CORE_SECTION_PATTERN, section_name):<EOL><INDENT>section = Section(section_name,<EOL>guess_plugin(section_name),<EOL>[(option_name, value)])<EOL>return {section.new_name: section.get_cfg_dict()}<EOL><DEDENT>else:<EOL><INDENT>if option_name.startswith(PLUGIN_PREFIX):<EOL><INDENT>return {section.new_name: section.get_cfg_dict() for section in enable_sections([], [(option_name, value)])}<EOL><DEDENT>else:<EOL><INDENT>return {'<STR_LIT>': Option('<STR_LIT>', option_name, value, CORE_SCHEMA).converted}<EOL><DEDENT><DEDENT>

:type value: str
:type key: str
:rtype: {str: obj}


if self._converted is None:<EOL><INDENT>self._converted = self.converter(self.name, self.value)<EOL><DEDENT>return self._converted<EOL>


if self._as_tuple is None:<EOL><INDENT>self._as_tuple = self.converted.items()[<NUM_LIT:0>]<EOL><DEDENT>return self._as_tuple<EOL>


if self._converter is None:<EOL><INDENT>try:<EOL><INDENT>self._converter = self.SPECIAL_CONVERTERS[self.plugin][self.name]<EOL><DEDENT>except KeyError:<EOL><INDENT>try:<EOL><INDENT>self._converter = self._get_scheme_converter()<EOL><DEDENT>except UnknownOption:<EOL><INDENT>self._converter = self.CONVERTERS_FOR_UNKNOWN.get(self.plugin, self.dummy_converter)<EOL><DEDENT><DEDENT><DEDENT>return self._converter<EOL>


@classmethod<EOL><INDENT>def from_multiple(cls, sections, parent_name=None, child_name=None, is_list=True):<DEDENT>

if len(sections) == <NUM_LIT:1>:<EOL><INDENT>return sections[<NUM_LIT:0>]<EOL><DEDENT>if parent_name:<EOL><INDENT>master_section = filter(lambda section: section.name == parent_name, sections)[<NUM_LIT:0>]<EOL>rest = filter(lambda section: section.name != parent_name, sections)<EOL><DEDENT>else:<EOL><INDENT>master_section = sections[<NUM_LIT:0>]<EOL>parent_name = master_section.name<EOL>rest = sections[<NUM_LIT:1>:]<EOL><DEDENT>child = {'<STR_LIT>': [section.get_cfg_dict(with_meta=False) for section in rest]} if is_listelse {child_name: cls._select_one(master_section, rest).get_cfg_dict(with_meta=False)}<EOL>master_section.merged_options.update(child)<EOL>return master_section<EOL>

:type parent_name: str
:type sections: list of Section


def _validate_description(self, description, field, value):

pass<EOL>


def _validate_values_description(self, values_description, field, value):

pass<EOL>