def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path):

<EOL>if (self._last_header > header_path and<EOL>Match(r'<STR_LIT>', clean_lines.elided[linenum - <NUM_LIT:1>])):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Check if a header is in alphabetical order with the previous header.

        Args:
          clean_lines: A CleansedLines instance containing the file.
          linenum: The number of the line to check.
          header_path: Canonicalized header to be checked.

        Returns:
          Returns true if the header is in alphabetical order.


error_message = ('<STR_LIT>' %<EOL>(self._TYPE_NAMES[header_type],<EOL>self._SECTION_NAMES[self._section]))<EOL>last_section = self._section<EOL>if header_type == _C_SYS_HEADER:<EOL><INDENT>if self._section <= self._C_SECTION:<EOL><INDENT>self._section = self._C_SECTION<EOL><DEDENT>else:<EOL><INDENT>self._last_header = '<STR_LIT>'<EOL>return error_message<EOL><DEDENT><DEDENT>elif header_type == _CPP_SYS_HEADER:<EOL><INDENT>if self._section <= self._CPP_SECTION:<EOL><INDENT>self._section = self._CPP_SECTION<EOL><DEDENT>else:<EOL><INDENT>self._last_header = '<STR_LIT>'<EOL>return error_message<EOL><DEDENT><DEDENT>elif header_type == _LIKELY_MY_HEADER:<EOL><INDENT>if self._section <= self._MY_H_SECTION:<EOL><INDENT>self._section = self._MY_H_SECTION<EOL><DEDENT>else:<EOL><INDENT>self._section = self._OTHER_H_SECTION<EOL><DEDENT><DEDENT>elif header_type == _POSSIBLE_MY_HEADER:<EOL><INDENT>if self._section <= self._MY_H_SECTION:<EOL><INDENT>self._section = self._MY_H_SECTION<EOL><DEDENT>else:<EOL><INDENT>self._section = self._OTHER_H_SECTION<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert header_type == _OTHER_HEADER<EOL>self._section = self._OTHER_H_SECTION<EOL><DEDENT>if last_section != self._section:<EOL><INDENT>self._last_header = '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT>'<EOL>

Returns a non-empty error message if the next header is out of order.

        This function also updates the internal state to be ready to check
        the next include.

        Args:
          header_type: One of the _XXX_HEADER constants defined above.

        Returns:
          The empty string if the header is in the right order, or an
          error message describing what's wrong.


self.output_format = output_format<EOL>


last_verbose_level = self.verbose_level<EOL>self.verbose_level = level<EOL>return last_verbose_level<EOL>

Sets the module's verbosity, and returns the previous setting.


self.counting = counting_style<EOL>


<EOL>self.filters = _DEFAULT_FILTERS[:]<EOL>self.AddFilters(filters)<EOL>

Sets the error-message filters.

        These filters are applied when deciding whether to emit a given
        error message.

        Args:
          filters: A string of comma-separated filters (eg "+whitespace/indent").
                   Each filter should start with + or -; else we die.

        Raises:
          ValueError: The comma-separated filters did not all start with '+' or '-'.
                      E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter"


for filt in filters.split('<STR_LIT:U+002C>'):<EOL><INDENT>clean_filt = filt.strip()<EOL>if clean_filt:<EOL><INDENT>self.filters.append(clean_filt)<EOL><DEDENT><DEDENT>for filt in self.filters:<EOL><INDENT>if not (filt.startswith('<STR_LIT:+>') or filt.startswith('<STR_LIT:->')):<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % filt)<EOL><DEDENT><DEDENT>

Adds more filters to the existing list of error-message filters.


self._filters_backup = self.filters[:]<EOL>

Saves the current filter list to backup storage.


self.filters = self._filters_backup[:]<EOL>


self.error_count = <NUM_LIT:0><EOL>self.errors_by_category = {}<EOL>

Sets the module's error statistic back to zero.


self.error_count += <NUM_LIT:1><EOL>if self.counting in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if self.counting != '<STR_LIT>':<EOL><INDENT>category = category.split('<STR_LIT:/>')[<NUM_LIT:0>]<EOL><DEDENT>if category not in self.errors_by_category:<EOL><INDENT>self.errors_by_category[category] = <NUM_LIT:0><EOL><DEDENT>self.errors_by_category[category] += <NUM_LIT:1><EOL><DEDENT>


for category, count in sorted(iteritems(self.errors_by_category)):<EOL><INDENT>self.PrintInfo('<STR_LIT>' %<EOL>(category, count))<EOL><DEDENT>if self.error_count > <NUM_LIT:0>:<EOL><INDENT>self.PrintInfo('<STR_LIT>' % self.error_count)<EOL><DEDENT>

Print a summary of errors by category, and the total.


self.in_a_function = True<EOL>self.lines_in_function = <NUM_LIT:0><EOL>self.current_function = function_name<EOL>

Start analyzing function body.

        Args:
          function_name: The name of the function being tracked.


if self.in_a_function:<EOL><INDENT>self.lines_in_function += <NUM_LIT:1><EOL><DEDENT>


def Check(self, error, filename, linenum):

if not self.in_a_function:<EOL><INDENT>return<EOL><DEDENT>if Match(r'<STR_LIT>', self.current_function):<EOL><INDENT>base_trigger = self._TEST_TRIGGER<EOL><DEDENT>else:<EOL><INDENT>base_trigger = self._NORMAL_TRIGGER<EOL><DEDENT>trigger = base_trigger * <NUM_LIT:2>**_VerboseLevel()<EOL>if self.lines_in_function > trigger:<EOL><INDENT>error_level = int(math.log(self.lines_in_function / base_trigger, <NUM_LIT:2>))<EOL>if error_level > <NUM_LIT:5>:<EOL><INDENT>error_level = <NUM_LIT:5><EOL><DEDENT>error(filename, linenum, '<STR_LIT>', error_level,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'  % (<EOL>self.current_function, self.lines_in_function, trigger))<EOL><DEDENT>

Report if too many lines in function body.

        Args:
          error: The function to call with any errors found.
          filename: The name of the current file.
          linenum: The number of the line to check.


self.in_a_function = False<EOL>


return os.path.abspath(self._filename).replace('<STR_LIT:\\>', '<STR_LIT:/>')<EOL>


fullname = self.FullName()<EOL>if os.path.exists(fullname):<EOL><INDENT>project_dir = os.path.dirname(fullname)<EOL>if _repository:<EOL><INDENT>repo = FileInfo(_repository).FullName()<EOL>root_dir = project_dir<EOL>while os.path.exists(root_dir):<EOL><INDENT>if os.path.normcase(root_dir) == os.path.normcase(repo):<EOL><INDENT>return os.path.relpath(fullname, root_dir).replace('<STR_LIT:\\>', '<STR_LIT:/>')<EOL><DEDENT>one_up_dir = os.path.dirname(root_dir)<EOL>if one_up_dir == root_dir:<EOL><INDENT>break<EOL><DEDENT>root_dir = one_up_dir<EOL><DEDENT><DEDENT>if os.path.exists(os.path.join(project_dir, "<STR_LIT>")):<EOL><INDENT>root_dir = project_dir<EOL>one_up_dir = os.path.dirname(root_dir)<EOL>while os.path.exists(os.path.join(one_up_dir, "<STR_LIT>")):<EOL><INDENT>root_dir = os.path.dirname(root_dir)<EOL>one_up_dir = os.path.dirname(one_up_dir)<EOL><DEDENT>prefix = os.path.commonprefix([root_dir, project_dir])<EOL>return fullname[len(prefix) + <NUM_LIT:1>:]<EOL><DEDENT>root_dir = current_dir = os.path.dirname(fullname)<EOL>while current_dir != os.path.dirname(current_dir):<EOL><INDENT>if (os.path.exists(os.path.join(current_dir, "<STR_LIT>")) or<EOL>os.path.exists(os.path.join(current_dir, "<STR_LIT>")) or<EOL>os.path.exists(os.path.join(current_dir, "<STR_LIT>"))):<EOL><INDENT>root_dir = current_dir<EOL><DEDENT>current_dir = os.path.dirname(current_dir)<EOL><DEDENT>if (os.path.exists(os.path.join(root_dir, "<STR_LIT>")) or<EOL>os.path.exists(os.path.join(root_dir, "<STR_LIT>")) or<EOL>os.path.exists(os.path.join(root_dir, "<STR_LIT>"))):<EOL><INDENT>prefix = os.path.commonprefix([root_dir, project_dir])<EOL>return fullname[len(prefix) + <NUM_LIT:1>:]<EOL><DEDENT><DEDENT>return fullname<EOL>

r"""FullName after removing the local path to the repository.

        If we have a real absolute path name here we can try to do something smart:
        detecting the root of the checkout and truncating /path/to/checkout from
        the name so that we get header guards that don't include things like
        "C:\Documents and Settings\..." or "/home/username/..." in them and thus
        people on different computers who have checked the source out to different
        locations won't see bogus errors.


googlename = self.RepositoryName()<EOL>project, rest = os.path.split(googlename)<EOL>return (project,) + os.path.splitext(rest)<EOL>

Splits the file into the directory, basename, and extension.

        For 'chrome/browser/browser.cc', Split() would
        return ('chrome/browser', 'browser', '.cc')

        Returns:
          A tuple of (directory, basename, extension).


return self.Split()[<NUM_LIT:1>]<EOL>

File base name - text after the final slash, before the final period.


return self.Split()[<NUM_LIT:2>]<EOL>

File extension - text following the final period, includes that period.


return '<STR_LIT:/>'.join(self.Split()[<NUM_LIT:0>:<NUM_LIT:2>])<EOL>


return _IsSourceExtension(self.Extension()[<NUM_LIT:1>:])<EOL>


return self.num_lines<EOL>

Returns the number of lines represented.


if _RE_PATTERN_INCLUDE.match(elided):<EOL><INDENT>return elided<EOL><DEDENT>elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('<STR_LIT>', elided)<EOL>collapsed = '<STR_LIT>'<EOL>while True:<EOL><INDENT>match = Match(r'<STR_LIT>', elided)<EOL>if not match:<EOL><INDENT>collapsed += elided<EOL>break<EOL><DEDENT>head, quote, tail = match.groups()<EOL>if quote == '<STR_LIT:">':<EOL><INDENT>second_quote = tail.find('<STR_LIT:">')<EOL>if second_quote >= <NUM_LIT:0>:<EOL><INDENT>collapsed += head + '<STR_LIT>'<EOL>elided = tail[second_quote + <NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>collapsed += elided<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if Search(r'<STR_LIT>', head):<EOL><INDENT>match_literal = Match(r'<STR_LIT>', "<STR_LIT:'>" + tail)<EOL>collapsed += head + match_literal.group(<NUM_LIT:1>).replace("<STR_LIT:'>", '<STR_LIT>')<EOL>elided = match_literal.group(<NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>second_quote = tail.find('<STR_LIT>')<EOL>if second_quote >= <NUM_LIT:0>:<EOL><INDENT>collapsed += head + "<STR_LIT>"<EOL>elided = tail[second_quote + <NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>collapsed += elided<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return collapsed<EOL>

Collapses strings and chars on a line to simple "" or '' blocks.

        We nix strings first so we're not fooled by text like '"http://"'

        Args:
          elided: The line being processed.

        Returns:
          The line with collapsed strings.


def CheckBegin(self, filename, clean_lines, linenum, error):

pass<EOL>

Run checks that applies to text up to the opening brace.

        This is mostly for checking the text after the class identifier
        and the "{", usually where the base class is specified.  For other
        blocks, there isn't much to check, so we always pass.

        Args:
          filename: The name of the current file.
          clean_lines: A CleansedLines instance containing the file.
          linenum: The number of the line to check.
          error: The function to call with any errors found.


def CheckEnd(self, filename, clean_lines, linenum, error):

pass<EOL>

Run checks that applies to text after the closing brace.

        This is mostly used for checking end of namespace comments.

        Args:
          filename: The name of the current file.
          clean_lines: A CleansedLines instance containing the file.
          linenum: The number of the line to check.
          error: The function to call with any errors found.


return self.__class__ == _BlockInfo<EOL>

Returns true if this block is a _BlockInfo.

        This is convenient for verifying that an object is an instance of
        a _BlockInfo, but not an instance of any of the derived classes.

        Returns:
          True for this class, False for derived classes.


line = clean_lines.raw_lines[linenum]<EOL>if (linenum - self.starting_linenum < <NUM_LIT:10><EOL>and not Match(r'<STR_LIT>', line)):<EOL><INDENT>return<EOL><DEDENT>if self.name:<EOL><INDENT>if not Match((r'<STR_LIT>' +<EOL>re.escape(self.name) + r'<STR_LIT>'),<EOL>line):<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' %<EOL>self.name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not Match(r'<STR_LIT>', line):<EOL><INDENT>if Match(r'<STR_LIT>', line):<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>


return (not self.stack) or self.stack[-<NUM_LIT:1>].seen_open_brace<EOL>

Check if we have seen the opening brace for the innermost block.

        Returns:
          True if we have seen the opening brace, False if the innermost
          block is still expecting an opening brace.


return self.stack and isinstance(self.stack[-<NUM_LIT:1>], _NamespaceInfo)<EOL>

Check if we are currently one level inside a namespace body.

        Returns:
          True if top of the stack is a namespace block, False otherwise.


return self.stack and isinstance(self.stack[-<NUM_LIT:1>], _ExternCInfo)<EOL>

Check if we are currently one level inside an 'extern "C"' block.

        Returns:
          True if top of the stack is an extern block, False otherwise.


return self.stack and isinstance(self.stack[-<NUM_LIT:1>], _ClassInfo)<EOL>

Check if we are currently one level inside a class or struct declaration.

        Returns:
          True if top of the stack is a class/struct, False otherwise.


return self.stack and self.stack[-<NUM_LIT:1>].inline_asm != _NO_ASM<EOL>

Check if we are currently one level inside an inline ASM block.

        Returns:
          True if the top of the stack is a block containing inline ASM.


def InTemplateArgumentList(self, clean_lines, linenum, pos):

while linenum < clean_lines.NumLines():<EOL><INDENT>line = clean_lines.elided[linenum]<EOL>match = Match(r'<STR_LIT>', line[pos:])<EOL>if not match:<EOL><INDENT>linenum += <NUM_LIT:1><EOL>pos = <NUM_LIT:0><EOL>continue<EOL><DEDENT>token = match.group(<NUM_LIT:1>)<EOL>pos += len(match.group(<NUM_LIT:0>))<EOL>if token in ('<STR_LIT:{>', '<STR_LIT:}>', '<STR_LIT:;>'): return False<EOL>if token in ('<STR_LIT:>>', '<STR_LIT:=>', '<STR_LIT:[>', '<STR_LIT:]>', '<STR_LIT:.>'): return True<EOL>if token != '<STR_LIT:<>':<EOL><INDENT>pos += <NUM_LIT:1><EOL>if pos >= len(line):<EOL><INDENT>linenum += <NUM_LIT:1><EOL>pos = <NUM_LIT:0><EOL><DEDENT>continue<EOL><DEDENT>(_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - <NUM_LIT:1>)<EOL>if end_pos < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>linenum = end_line<EOL>pos = end_pos<EOL><DEDENT>return False<EOL>

Check if current position is inside template argument list.

        Args:
          clean_lines: A CleansedLines instance containing the file.
          linenum: The number of the line to check.
          pos: position just after the suspected template argument.
        Returns:
          True if (linenum, pos) is inside template arguments.


if Match(r'<STR_LIT>', line):<EOL><INDENT>self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack)))<EOL><DEDENT>elif Match(r'<STR_LIT>', line):<EOL><INDENT>if self.pp_stack:<EOL><INDENT>if not self.pp_stack[-<NUM_LIT:1>].seen_else:<EOL><INDENT>self.pp_stack[-<NUM_LIT:1>].seen_else = True<EOL>self.pp_stack[-<NUM_LIT:1>].stack_before_else = copy.deepcopy(self.stack)<EOL><DEDENT>self.stack = copy.deepcopy(self.pp_stack[-<NUM_LIT:1>].stack_before_if)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>elif Match(r'<STR_LIT>', line):<EOL><INDENT>if self.pp_stack:<EOL><INDENT>if self.pp_stack[-<NUM_LIT:1>].seen_else:<EOL><INDENT>self.stack = self.pp_stack[-<NUM_LIT:1>].stack_before_else<EOL><DEDENT>self.pp_stack.pop()<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>

Update preprocessor stack.

        We need to handle preprocessors due to classes like this:
          #ifdef SWIG
          struct ResultDetailsPageElementExtensionPoint {
          #else
          struct ResultDetailsPageElementExtensionPoint : public Extension {
          #endif

        We make the following assumptions (good enough for most files):
        - Preprocessor condition evaluates to true from #if up to first
          #else/#elif/#endif.

        - Preprocessor condition evaluates to false from #else/#elif up
          to #endif.  We still perform lint checks on these lines, but
          these do not affect nesting stack.

        Args:
          line: current line to check.


def Update(self, filename, clean_lines, linenum, error):

line = clean_lines.elided[linenum]<EOL>if self.stack:<EOL><INDENT>self.previous_stack_top = self.stack[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>self.previous_stack_top = None<EOL><DEDENT>self.UpdatePreprocessor(line)<EOL>if self.stack:<EOL><INDENT>inner_block = self.stack[-<NUM_LIT:1>]<EOL>depth_change = line.count('<STR_LIT:(>') - line.count('<STR_LIT:)>')<EOL>inner_block.open_parentheses += depth_change<EOL>if inner_block.inline_asm in (_NO_ASM, _END_ASM):<EOL><INDENT>if (depth_change != <NUM_LIT:0> and<EOL>inner_block.open_parentheses == <NUM_LIT:1> and<EOL>_MATCH_ASM.match(line)):<EOL><INDENT>inner_block.inline_asm = _INSIDE_ASM<EOL><DEDENT>else:<EOL><INDENT>inner_block.inline_asm = _NO_ASM<EOL><DEDENT><DEDENT>elif (inner_block.inline_asm == _INSIDE_ASM and<EOL>inner_block.open_parentheses == <NUM_LIT:0>):<EOL><INDENT>inner_block.inline_asm = _END_ASM<EOL><DEDENT><DEDENT>while True:<EOL><INDENT>namespace_decl_match = Match(r'<STR_LIT>', line)<EOL>if not namespace_decl_match:<EOL><INDENT>break<EOL><DEDENT>new_namespace = _NamespaceInfo(namespace_decl_match.group(<NUM_LIT:1>), linenum)<EOL>self.stack.append(new_namespace)<EOL>line = namespace_decl_match.group(<NUM_LIT:2>)<EOL>if line.find('<STR_LIT:{>') != -<NUM_LIT:1>:<EOL><INDENT>new_namespace.seen_open_brace = True<EOL>line = line[line.find('<STR_LIT:{>') + <NUM_LIT:1>:]<EOL><DEDENT><DEDENT>class_decl_match = Match(<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>', line)<EOL>if (class_decl_match and<EOL>(not self.stack or self.stack[-<NUM_LIT:1>].open_parentheses == <NUM_LIT:0>)):<EOL><INDENT>end_declaration = len(class_decl_match.group(<NUM_LIT:1>))<EOL>if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration):<EOL><INDENT>self.stack.append(_ClassInfo(<EOL>class_decl_match.group(<NUM_LIT:3>), class_decl_match.group(<NUM_LIT:2>),<EOL>clean_lines, linenum))<EOL>line = class_decl_match.group(<NUM_LIT:4>)<EOL><DEDENT><DEDENT>if not self.SeenOpenBrace():<EOL><INDENT>self.stack[-<NUM_LIT:1>].CheckBegin(filename, clean_lines, linenum, error)<EOL><DEDENT>if self.stack and isinstance(self.stack[-<NUM_LIT:1>], _ClassInfo):<EOL><INDENT>classinfo = self.stack[-<NUM_LIT:1>]<EOL>access_match = Match(<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>',<EOL>line)<EOL>if access_match:<EOL><INDENT>classinfo.access = access_match.group(<NUM_LIT:2>)<EOL>indent = access_match.group(<NUM_LIT:1>)<EOL>if (len(indent) != classinfo.class_indent + <NUM_LIT:1> and<EOL>Match(r'<STR_LIT>', indent)):<EOL><INDENT>if classinfo.is_struct:<EOL><INDENT>parent = '<STR_LIT>' + classinfo.name<EOL><DEDENT>else:<EOL><INDENT>parent = '<STR_LIT>' + classinfo.name<EOL><DEDENT>slots = '<STR_LIT>'<EOL>if access_match.group(<NUM_LIT:3>):<EOL><INDENT>slots = access_match.group(<NUM_LIT:3>)<EOL><DEDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:3>,<EOL>'<STR_LIT>' % (<EOL>access_match.group(<NUM_LIT:2>), slots, parent))<EOL><DEDENT><DEDENT><DEDENT>while True:<EOL><INDENT>matched = Match(r'<STR_LIT>', line)<EOL>if not matched:<EOL><INDENT>break<EOL><DEDENT>token = matched.group(<NUM_LIT:1>)<EOL>if token == '<STR_LIT:{>':<EOL><INDENT>if not self.SeenOpenBrace():<EOL><INDENT>self.stack[-<NUM_LIT:1>].seen_open_brace = True<EOL><DEDENT>elif Match(r'<STR_LIT>', line):<EOL><INDENT>self.stack.append(_ExternCInfo(linenum))<EOL><DEDENT>else:<EOL><INDENT>self.stack.append(_BlockInfo(linenum, True))<EOL>if _MATCH_ASM.match(line):<EOL><INDENT>self.stack[-<NUM_LIT:1>].inline_asm = _BLOCK_ASM<EOL><DEDENT><DEDENT><DEDENT>elif token == '<STR_LIT:;>' or token == '<STR_LIT:)>':<EOL><INDENT>if not self.SeenOpenBrace():<EOL><INDENT>self.stack.pop()<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>if self.stack:<EOL><INDENT>self.stack[-<NUM_LIT:1>].CheckEnd(filename, clean_lines, linenum, error)<EOL>self.stack.pop()<EOL><DEDENT><DEDENT>line = matched.group(<NUM_LIT:2>)<EOL><DEDENT>

Update nesting state with current line.

        Args:
          filename: The name of the current file.
          clean_lines: A CleansedLines instance containing the file.
          linenum: The number of the line to check.
          error: The function to call with any errors found.


for i in range(len(self.stack), <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>classinfo = self.stack[i - <NUM_LIT:1>]<EOL>if isinstance(classinfo, _ClassInfo):<EOL><INDENT>return classinfo<EOL><DEDENT><DEDENT>return None<EOL>

Get class info on the top of the stack.

        Returns:
          A _ClassInfo object if we are inside a class, or None otherwise.


def CheckCompletedBlocks(self, filename, error):

<EOL>for obj in self.stack:<EOL><INDENT>if isinstance(obj, _ClassInfo):<EOL><INDENT>error(filename, obj.starting_linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' %<EOL>obj.name)<EOL><DEDENT>elif isinstance(obj, _NamespaceInfo):<EOL><INDENT>error(filename, obj.starting_linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' %<EOL>obj.name)<EOL><DEDENT><DEDENT>

Checks that all classes and namespaces have been completely parsed.

        Call this when all lines in a file have been processed.
        Args:
          filename: The name of the current file.
          error: The function to call with any errors found.


match = _REGEX.match(version)<EOL>if match is None:<EOL><INDENT>raise ValueError('<STR_LIT>' % version)<EOL><DEDENT>verinfo = match.groupdict()<EOL>verinfo['<STR_LIT>'] = int(verinfo['<STR_LIT>'])<EOL>verinfo['<STR_LIT>'] = int(verinfo['<STR_LIT>'])<EOL>verinfo['<STR_LIT>'] = int(verinfo['<STR_LIT>'])<EOL>return verinfo<EOL>

Parse version to major, minor, patch, pre-release, build parts.


def submit_topology(heron_cli_path, cli_config_path, cluster, role,<EOL>env, jar_path, classpath, pkg_uri, args=None):

cmd = "<STR_LIT>" %(heron_cli_path, cli_config_path, cluster_token(cluster, role, env),<EOL>jar_path, classpath, args)<EOL>if pkg_uri is not None:<EOL><INDENT>cmd = "<STR_LIT>" %(cmd, pkg_uri)<EOL><DEDENT>logging.info("<STR_LIT>", cmd)<EOL>if os.system(cmd) != <NUM_LIT:0>:<EOL><INDENT>raise status.TestFailure("<STR_LIT>")<EOL><DEDENT>


def kill_topology(heron_cli_path, cli_config_path, cluster, role, env, topology_name):

cmd = "<STR_LIT>" %(heron_cli_path, cli_config_path, cluster_token(cluster, role, env), topology_name)<EOL>logging.info("<STR_LIT>", cmd)<EOL>if os.system(cmd) != <NUM_LIT:0>:<EOL><INDENT>raise status.TestFailure("<STR_LIT>" % topology_name)<EOL><DEDENT>logging.info("<STR_LIT>", topology_name)<EOL>


log.configure(level=logging.DEBUG)<EOL>conf_file = DEFAULT_TEST_CONF_FILE<EOL>conf_string = pkgutil.get_data(__name__, conf_file)<EOL>decoder = json.JSONDecoder(strict=False)<EOL>conf = decoder.decode(conf_string)<EOL>parser = argparse.ArgumentParser(description='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default='<STR_LIT:localhost>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', type=int,<EOL>default='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=None)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=None)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=<NUM_LIT:1>)<EOL>args, unknown_args = parser.parse_known_args()<EOL>if unknown_args:<EOL><INDENT>logging.error('<STR_LIT>', sys.argv[<NUM_LIT:0>], unknown_args[<NUM_LIT:0>])<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>tests_start_time = int(time.time())<EOL>run_topology_tests(conf, args)<EOL>total = len(failures) + len(successes)<EOL>logging.info("<STR_LIT>" % (int(time.time()) - tests_start_time))<EOL>if not failures:<EOL><INDENT>logging.info("<STR_LIT>", len(successes))<EOL>for test in successes:<EOL><INDENT>logging.info("<STR_LIT>", ("<STR_LIT>" % test[<NUM_LIT:1>]).ljust(<NUM_LIT:8>), test[<NUM_LIT:0>])<EOL><DEDENT>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>logging.error("<STR_LIT>", len(failures), total)<EOL>for test in failures:<EOL><INDENT>logging.error("<STR_LIT>", ("<STR_LIT>" % test[<NUM_LIT:1>]).ljust(<NUM_LIT:8>), test[<NUM_LIT:0>])<EOL><DEDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>


expected_result = self.topology_structure_expected_results_handler.fetch_results()<EOL>actual_result = self.topology_structure_actual_results_handler.fetch_cur_pplan()<EOL>self.topology_structure_actual_results_handler.stop_state_mgr()<EOL>decoder = json.JSONDecoder(strict=False)<EOL>expected_results = decoder.decode(expected_result)<EOL>return self._compare(expected_results, actual_result)<EOL>

Checks the topology graph structure from zk with the expected results from local file


def _compare(self, expected_results, actual_results):

expected_nodes, expected_links = self._parse_expected_results(expected_results)<EOL>actual_nodes, actual_links = self._parse_actual_results(actual_results)<EOL>correct_topology = True<EOL>if correct_topology:<EOL><INDENT>for key_expected in expected_nodes:<EOL><INDENT>if key_expected not in actual_nodes:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT>if expected_nodes[key_expected] != actual_nodes[key_expected]:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if correct_topology:<EOL><INDENT>for key_actual in actual_nodes:<EOL><INDENT>if key_actual not in expected_nodes:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT>if expected_nodes[key_actual] != actual_nodes[key_actual]:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if correct_topology:<EOL><INDENT>for key_expected in expected_links:<EOL><INDENT>if key_expected not in actual_links:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT>if expected_links[key_expected] != actual_links[key_expected]:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if correct_topology:<EOL><INDENT>for key_actual in actual_links:<EOL><INDENT>if key_actual not in expected_links:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT>if expected_links[key_actual] != actual_links[key_actual]:<EOL><INDENT>correct_topology = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if correct_topology:<EOL><INDENT>return status.TestSuccess(<EOL>"<STR_LIT>" % self.topology_name)<EOL><DEDENT>else:<EOL><INDENT>raise status.TestFailure("<STR_LIT>")<EOL><DEDENT>

check if the topology structure is correct


expected_nodes = dict()<EOL>expected_links = dict()<EOL>for bolt in expected_results["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>name = bolt["<STR_LIT>"]["<STR_LIT:name>"]<EOL>if name not in expected_links:<EOL><INDENT>expected_links[name] = set()<EOL><DEDENT>for input in bolt["<STR_LIT>"]:<EOL><INDENT>expected_links[name].add(input["<STR_LIT>"]["<STR_LIT>"])<EOL><DEDENT><DEDENT>for instance in expected_results["<STR_LIT>"]:<EOL><INDENT>name = instance["<STR_LIT:info>"]["<STR_LIT>"]<EOL>if name not in expected_nodes:<EOL><INDENT>expected_nodes[name] = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>expected_nodes[name] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return  expected_nodes, expected_links<EOL>

Parse JSON file and generate expected_nodes and expected_links


actual_nodes = dict()<EOL>actual_links = dict()<EOL>for bolt in actual_results.topology.bolts:<EOL><INDENT>name = bolt.comp.name<EOL>if name not in actual_links:<EOL><INDENT>actual_links[name] = set()<EOL><DEDENT>for input in bolt.inputs:<EOL><INDENT>actual_links[name].add(input.stream.component_name)<EOL><DEDENT><DEDENT>for instance in actual_results.instances:<EOL><INDENT>name = instance.info.component_name<EOL>if name not in actual_nodes:<EOL><INDENT>actual_nodes[name] = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>actual_nodes[name] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return actual_nodes, actual_links<EOL>

Parse protobuf messege and generate actual_nodes and actual_links


try:<EOL><INDENT>if not os.path.exists(self.file_path):<EOL><INDENT>raise status.TestFailure("<STR_LIT>" % self.file_path)<EOL><DEDENT>else:<EOL><INDENT>with open(self.file_path, "<STR_LIT:r>") as expected_result_file:<EOL><INDENT>return expected_result_file.read().rstrip()<EOL><DEDENT><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>raise status.TestFailure("<STR_LIT>" % self.file_path, e)<EOL><DEDENT>

Read expected result from the expected result file


def fetch_from_server(self, server_host_port, topology_name, data_name, path):

for i in range(<NUM_LIT:0>, RETRY_ATTEMPTS):<EOL><INDENT>logging.info("<STR_LIT>", data_name, topology_name, i)<EOL>response = self.get_http_response(server_host_port, path)<EOL>if response.status == <NUM_LIT:200>:<EOL><INDENT>return response.read()<EOL><DEDENT>elif i != RETRY_ATTEMPTS:<EOL><INDENT>logging.info("<STR_LIT>",<EOL>data_name, response.status, response.reason, response.read())<EOL>time.sleep(RETRY_INTERVAL)<EOL><DEDENT><DEDENT>raise status.TestFailure("<STR_LIT>" % (data_name, RETRY_ATTEMPTS))<EOL>

Make a http get request to fetch actual results from http server


def get_http_response(self, server_host_port, path):

for _ in range(<NUM_LIT:0>, RETRY_ATTEMPTS):<EOL><INDENT>try:<EOL><INDENT>connection = HTTPConnection(server_host_port)<EOL>connection.request('<STR_LIT:GET>', path)<EOL>response = connection.getresponse()<EOL>return response<EOL><DEDENT>except Exception:<EOL><INDENT>time.sleep(RETRY_INTERVAL)<EOL>continue<EOL><DEDENT><DEDENT>raise status.TestFailure("<STR_LIT>" % RETRY_ATTEMPTS)<EOL>


def fetch_from_server(server_host_port, topology_name, data_name, path):

for i in range(<NUM_LIT:0>, RETRY_ATTEMPTS):<EOL><INDENT>logging.info("<STR_LIT>", data_name, topology_name, i)<EOL>response = get_http_response(server_host_port, path)<EOL>if response.status == <NUM_LIT:200>:<EOL><INDENT>return response.read()<EOL><DEDENT>elif i != RETRY_ATTEMPTS:<EOL><INDENT>logging.info("<STR_LIT>",<EOL>data_name, response.status, response.reason, response.read())<EOL>time.sleep(RETRY_INTERVAL)<EOL><DEDENT><DEDENT>raise status.TestFailure("<STR_LIT>" % (data_name, RETRY_ATTEMPTS))<EOL>


<EOL>cmd = "<STR_LIT>" %(heron_cli_path, cli_config_path, cluster_token(cluster, role, env),<EOL>jar_path, classpath, args)<EOL>if pkg_uri is not None:<EOL><INDENT>cmd = "<STR_LIT>" %(cmd, pkg_uri)<EOL><DEDENT>logging.info("<STR_LIT>", cmd)<EOL>if os.system(cmd) != <NUM_LIT:0>:<EOL><INDENT>raise status.TestFailure("<STR_LIT>")<EOL><DEDENT>


log.configure(level=logging.DEBUG)<EOL>conf_file = DEFAULT_TEST_CONF_FILE<EOL>conf_string = pkgutil.get_data(__name__, conf_file)<EOL>decoder = json.JSONDecoder(strict=False)<EOL>conf = decoder.decode(conf_string)<EOL>parser = argparse.ArgumentParser(description='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', type=int,<EOL>default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=None)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=None)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>',<EOL>default=conf['<STR_LIT>'])<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', default=<NUM_LIT:1>)<EOL>args, unknown_args = parser.parse_known_args()<EOL>if unknown_args:<EOL><INDENT>logging.error('<STR_LIT>', sys.argv[<NUM_LIT:0>], unknown_args[<NUM_LIT:0>])<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>tests_start_time = int(time.time())<EOL>run_tests(conf, args)<EOL>total = len(failures) + len(successes)<EOL>logging.info("<STR_LIT>" % (int(time.time()) - tests_start_time))<EOL>if not failures:<EOL><INDENT>logging.info("<STR_LIT>", len(successes))<EOL>for test in successes:<EOL><INDENT>logging.info("<STR_LIT>", ("<STR_LIT>" % test[<NUM_LIT:1>]).ljust(<NUM_LIT:8>), test[<NUM_LIT:0>])<EOL><DEDENT>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>logging.error("<STR_LIT>", len(failures), total)<EOL>for test in failures:<EOL><INDENT>logging.error("<STR_LIT>", ("<STR_LIT>" % test[<NUM_LIT:1>]).ljust(<NUM_LIT:8>), test[<NUM_LIT:0>])<EOL><DEDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>


actual_result = self.actual_results_handler.fetch_results()<EOL>expected_result = self.expected_results_handler.fetch_results()<EOL>decoder = json.JSONDecoder(strict=False)<EOL>actual_results = sorted(decoder.decode(actual_result))<EOL>expected_results = sorted(decoder.decode(expected_result))<EOL>return self._compare(expected_results, actual_results)<EOL>

Checks the topology results from the server with the expected results from the file


log.configure(level=logging.DEBUG)<EOL>conf_file = DEFAULT_TEST_CONF_FILE<EOL>conf_string = pkgutil.get_data(__name__, conf_file)<EOL>decoder = json.JSONDecoder(strict=False)<EOL>conf = decoder.decode(conf_string)<EOL>args = dict()<EOL>home_directory = os.path.expanduser("<STR_LIT>")<EOL>args['<STR_LIT>'] = conf['<STR_LIT>']<EOL>args['<STR_LIT>'] = conf['<STR_LIT>']['<STR_LIT>']<EOL>args['<STR_LIT>'] = conf['<STR_LIT>']['<STR_LIT>']<EOL>args['<STR_LIT>'] = os.path.join(<EOL>home_directory,<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>conf['<STR_LIT>'],<EOL>getpass.getuser(),<EOL>args['<STR_LIT>']<EOL>)<EOL>args['<STR_LIT>'] = os.path.expanduser(conf['<STR_LIT>'])<EOL>args['<STR_LIT>'] = os.path.expanduser(conf['<STR_LIT>'])<EOL>args['<STR_LIT>'] = _random_port()<EOL>args['<STR_LIT>'] = os.path.join(args['<STR_LIT>'], conf['<STR_LIT>']['<STR_LIT>'])<EOL>args['<STR_LIT>'] = os.path.join(args['<STR_LIT>'], conf['<STR_LIT>']['<STR_LIT>'])<EOL>args['<STR_LIT>'] = conf['<STR_LIT>']<EOL>test_classes = TEST_CLASSES<EOL>if len(sys.argv) > <NUM_LIT:1>:<EOL><INDENT>first_arg = sys.argv[<NUM_LIT:1>]<EOL>class_tokens = first_arg.split("<STR_LIT:.>")<EOL>if first_arg == "<STR_LIT>" or len(class_tokens) < <NUM_LIT:2>:<EOL><INDENT>usage()<EOL><DEDENT>import importlib<EOL>package_tokens = class_tokens[:-<NUM_LIT:1>]<EOL>test_class = class_tokens[-<NUM_LIT:1>]<EOL>if len(package_tokens) == <NUM_LIT:1>: <EOL><INDENT>test_module = "<STR_LIT>" + package_tokens[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>test_module = '<STR_LIT:.>'.join(package_tokens)<EOL><DEDENT>logging.info("<STR_LIT>", test_module)<EOL>logging.info("<STR_LIT>", test_class)<EOL>test_classes = [getattr(importlib.import_module(test_module), test_class)]<EOL><DEDENT>start_time = time.time()<EOL>(successes, failures) = run_tests(test_classes, args)<EOL>elapsed_time = time.time() - start_time<EOL>total = len(failures) + len(successes)<EOL>if not failures:<EOL><INDENT>logging.info("<STR_LIT>", len(successes))<EOL>logging.info("<STR_LIT>", elapsed_time)<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>logging.error("<STR_LIT>", len(failures), total)<EOL>for test in failures:<EOL><INDENT>logging.error("<STR_LIT>", test)<EOL><DEDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>


<EOL>processes = subprocess.check_output(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>processes = processes.split('<STR_LIT:\n>')<EOL>processes = processes[<NUM_LIT:1>:] <EOL>process_list = []<EOL>for process in processes:<EOL><INDENT>if process == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>pretuple = process.split('<STR_LIT:U+0020>', <NUM_LIT:1>)<EOL>process_list.append(ProcessTuple(pretuple[<NUM_LIT:0>], pretuple[<NUM_LIT:1>]))<EOL><DEDENT>return process_list<EOL>

returns a list of process tuples (pid, cmd)
This only applies only for local scheduler as it uses the ps command
and assumes the topology will be running on different processes on same machine


def _processes_exists(process_list, process_cmd, min_processes):

proccess_count = <NUM_LIT:0><EOL>for process in process_list:<EOL><INDENT>if process_cmd in process.cmd:<EOL><INDENT>proccess_count += <NUM_LIT:1><EOL><DEDENT><DEDENT>return proccess_count >= min_processes<EOL>


expected_result = "<STR_LIT>"<EOL>actual_result = "<STR_LIT>"<EOL>retries_left = RETRY_COUNT<EOL>_sleep("<STR_LIT>" % self.testname, RETRY_INTERVAL)<EOL>while retries_left > <NUM_LIT:0>:<EOL><INDENT>retries_left -= <NUM_LIT:1><EOL>try:<EOL><INDENT>with open(self.params['<STR_LIT>'], '<STR_LIT:r>') as f:<EOL><INDENT>expected_result = f.read()<EOL><DEDENT>with open(self.params['<STR_LIT>'], '<STR_LIT:r>') as g:<EOL><INDENT>actual_result = g.read()<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>message ="<STR_LIT>" % self.testname<EOL>if retries_left == <NUM_LIT:0>:<EOL><INDENT>raise status.TestFailure(message, e)<EOL><DEDENT>logging.error(message, e)<EOL><DEDENT>expected_sorted = sorted(expected_result.split('<STR_LIT:\n>'))<EOL>actual_sorted = sorted(actual_result.split('<STR_LIT:\n>'))<EOL>if expected_sorted == actual_sorted:<EOL><INDENT>break<EOL><DEDENT>if retries_left > <NUM_LIT:0>:<EOL><INDENT>expected_result = "<STR_LIT>"<EOL>actual_result = "<STR_LIT>"<EOL>expected_sorted = []<EOL>actual_sorted = []<EOL>logging.info("<STR_LIT>"+ "<STR_LIT>",<EOL>self.testname, RETRY_COUNT - retries_left, RETRY_COUNT, RETRY_INTERVAL)<EOL>time.sleep(RETRY_INTERVAL)<EOL><DEDENT><DEDENT>if actual_sorted == expected_sorted:<EOL><INDENT>success = status.TestSuccess(<EOL>"<STR_LIT>" % self.testname)<EOL>logging.info("<STR_LIT>", actual_sorted)<EOL>logging.info("<STR_LIT>", expected_sorted)<EOL>return success<EOL><DEDENT>else:<EOL><INDENT>failure = status.TestFailure(<EOL>"<STR_LIT>" % self.testname)<EOL>logging.info("<STR_LIT>", actual_sorted)<EOL>logging.info("<STR_LIT>", expected_sorted)<EOL>raise failure<EOL><DEDENT>

get actual and expected result.
        retry if results are not equal a predesignated amount of times


def get_pid(self, process_name, heron_working_directory):

process_pid_file = os.path.join(heron_working_directory, process_name + '<STR_LIT>')<EOL>try:<EOL><INDENT>with open(process_pid_file, '<STR_LIT:r>') as f:<EOL><INDENT>pid = f.readline()<EOL>return pid<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>logging.error("<STR_LIT>", process_pid_file)<EOL>return -<NUM_LIT:1><EOL><DEDENT>

opens .pid file of process and reads the first and only line, which should be the process pid
if fail, return -1


if process_number < <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>" % process_number)<EOL><DEDENT>logging.info("<STR_LIT>", process_number)<EOL>try:<EOL><INDENT>os.kill(int(process_number), signal.SIGTERM)<EOL><DEDENT>except OSError as ex:<EOL><INDENT>if "<STR_LIT>" in str(ex): <EOL><INDENT>logging.info(str(ex))<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % process_number)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % process_number)<EOL><DEDENT>logging.info("<STR_LIT>", process_number)<EOL>

kills process by running unix command kill


def emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID,<EOL>direct_task=None, need_task_ids=None):

<EOL>self.tuples_to_complete += <NUM_LIT:1><EOL>if tup_id is None:<EOL><INDENT>Log.info("<STR_LIT>" % str(tup))<EOL>_tup_id = integ_const.INTEGRATION_TEST_MOCK_MESSAGE_ID<EOL><DEDENT>else:<EOL><INDENT>_tup_id = tup_id<EOL><DEDENT>super(IntegrationTestSpout, self).emit(tup, _tup_id, stream, direct_task, need_task_ids)<EOL>

Emits from this integration test spout

        Overriden method which will be called when user's spout calls emit()


def add_spout(self, name, spout_cls, par, config=None,<EOL>optional_outputs=None, max_executions=None):

user_spec = spout_cls.spec(name)<EOL>spout_classpath = user_spec.python_class_path<EOL>if hasattr(spout_cls, '<STR_LIT>'):<EOL><INDENT>user_outputs = spout_cls.outputs<EOL><DEDENT>else:<EOL><INDENT>user_outputs = []<EOL><DEDENT>if optional_outputs is not None:<EOL><INDENT>user_outputs.extend(optional_outputs)<EOL><DEDENT>if config is None:<EOL><INDENT>_config = {}<EOL><DEDENT>else:<EOL><INDENT>_config = copy.copy(config)<EOL><DEDENT>if max_executions is not None:<EOL><INDENT>_config[integ_const.USER_MAX_EXECUTIONS] = max_executions<EOL><DEDENT>test_spec = IntegrationTestSpout.spec(name, par, _config,<EOL>user_spout_classpath=spout_classpath,<EOL>user_output_fields=user_outputs)<EOL>self.add_spec(test_spec)<EOL>self.spouts[name] = test_spec<EOL>return test_spec<EOL>


def add_bolt(self, name, bolt_cls, par, inputs, config=None, optional_outputs=None):

assert isinstance(inputs, dict)<EOL>user_spec = bolt_cls.spec(name)<EOL>bolt_classpath = user_spec.python_class_path<EOL>if hasattr(bolt_cls, '<STR_LIT>'):<EOL><INDENT>user_outputs = bolt_cls.outputs<EOL><DEDENT>else:<EOL><INDENT>user_outputs = []<EOL><DEDENT>if optional_outputs is not None:<EOL><INDENT>user_outputs.extend(optional_outputs)<EOL><DEDENT>if config is None:<EOL><INDENT>_config = {}<EOL><DEDENT>else:<EOL><INDENT>_config = config<EOL><DEDENT>test_spec = IntegrationTestBolt.spec(name, par, inputs, _config,<EOL>user_bolt_classpath=bolt_classpath,<EOL>user_output_fields=user_outputs)<EOL>self.add_spec(test_spec)<EOL>self.bolts[name] = test_spec<EOL>return test_spec<EOL>

Add an integration_test bolt

        Only dict based inputs is supported


<EOL>aggregator_config = {integ_const.HTTP_POST_URL_KEY: self.output_location}<EOL>self.add_bolt(self.TERMINAL_BOLT_NAME, self.TERMINAL_BOLT_CLASS, <NUM_LIT:1>,<EOL>inputs={}, config=aggregator_config)<EOL>for name, bolt_spec in self.bolts.iteritems():<EOL><INDENT>if name == self.TERMINAL_BOLT_NAME:<EOL><INDENT>continue<EOL><DEDENT>bolt_protobuf = bolt_spec.get_protobuf()<EOL>for istream in bolt_protobuf.inputs:<EOL><INDENT>parent = istream.stream.component_name<EOL>if name in self.prev:<EOL><INDENT>self.prev[name].add(parent)<EOL><DEDENT>else:<EOL><INDENT>parents = set()<EOL>parents.add(parent)<EOL>self.prev[name] = parents<EOL><DEDENT><DEDENT><DEDENT>terminals = set()<EOL>non_terminals = set()<EOL>for parent_set in self.prev.values():<EOL><INDENT>non_terminals.update(parent_set)<EOL><DEDENT>for bolt_name in self.prev.keys():<EOL><INDENT>if bolt_name not in non_terminals:<EOL><INDENT>terminals.add(bolt_name)<EOL><DEDENT><DEDENT>for spout_name in self.spouts.keys():<EOL><INDENT>if spout_name not in non_terminals:<EOL><INDENT>terminals.add(spout_name)<EOL><DEDENT><DEDENT>for child in self.prev.keys():<EOL><INDENT>for parent in self.prev[child]:<EOL><INDENT>self._add_all_grouping(child, parent, integ_const.INTEGRATION_TEST_CONTROL_STREAM_ID)<EOL><DEDENT><DEDENT>for terminal in terminals:<EOL><INDENT>if terminal in self.bolts:<EOL><INDENT>terminal_outputs = self.bolts[terminal].outputs<EOL><DEDENT>else:<EOL><INDENT>terminal_outputs = self.spouts[terminal].outputs<EOL><DEDENT>stream_ids = ["<STR_LIT:default>" if isinstance(out, str) else out.stream_id<EOL>for out in terminal_outputs]<EOL>for stream_id in set(stream_ids):<EOL><INDENT>self._add_all_grouping(self.TERMINAL_BOLT_NAME, terminal, stream_id)<EOL><DEDENT><DEDENT>class_dict = self._construct_topo_class_dict()<EOL>return TopologyType(self.topology_name, (Topology,), class_dict)<EOL>

Creates an integration_test topology class


def _add_all_grouping(self, child, parent, stream_id):

<EOL>child_component_spec = self.bolts[child]<EOL>child_inputs = child_component_spec.inputs<EOL>if parent in self.bolts:<EOL><INDENT>parent_component_spec = self.bolts[parent]<EOL><DEDENT>else:<EOL><INDENT>parent_component_spec = self.spouts[parent]<EOL><DEDENT>if stream_id == Stream.DEFAULT_STREAM_ID:<EOL><INDENT>child_inputs[parent_component_spec] = Grouping.ALL<EOL><DEDENT>else:<EOL><INDENT>child_inputs[parent_component_spec[stream_id]] = Grouping.ALL<EOL><DEDENT>

Adds all grouping between child component and parent component with a given stream id

        :type child: str
        :param child: child's component name
        :type parent: str
        :param parent: parent's component name
        :type stream_id: str
        :param stream_id: stream id


builder = TestTopologyBuilder(topology_name, http_server_url)<EOL>ab_spout = builder.add_spout("<STR_LIT>", ABSpout, <NUM_LIT:1>)<EOL>builder.add_bolt("<STR_LIT>", IdentityBolt,<EOL>inputs={ab_spout: Grouping.ALL},<EOL>par=<NUM_LIT:2>,<EOL>optional_outputs=['<STR_LIT>'])<EOL>return builder.create_topology()<EOL>

Integration test topology builder for all grouping


builder = TestTopologyBuilder(topology_name, http_server_url)<EOL>ab_spout = builder.add_spout("<STR_LIT>", ABSpout, <NUM_LIT:1>)<EOL>builder.add_bolt("<STR_LIT>", IdentityBolt,<EOL>inputs={ab_spout: Grouping.NONE},<EOL>par=<NUM_LIT:3>,<EOL>optional_outputs=['<STR_LIT>'])<EOL>return builder.create_topology()<EOL>

Integration test topology builder for none grouping


builder = TestTopologyBuilder(topology_name, http_server_url)<EOL>ab_spout = builder.add_spout("<STR_LIT>", ABSpout, <NUM_LIT:1>)<EOL>builder.add_bolt("<STR_LIT>", IdentityBolt,<EOL>inputs={ab_spout: Grouping.SHUFFLE},<EOL>par=<NUM_LIT:1>,<EOL>optional_outputs=['<STR_LIT>'])<EOL>return builder.create_topology()<EOL>

Integration test topology builder for basic one task


log.configure(level=logging.DEBUG)<EOL>if not os.path.exists(RESULTS_DIRECTORY):<EOL><INDENT>os.makedirs(RESULTS_DIRECTORY)<EOL><DEDENT>state_map = {}<EOL>state_result_map = {}<EOL>application = tornado.web.Application([<EOL>(r"<STR_LIT:/>", MainHandler),<EOL>(r"<STR_LIT>", FileHandler),<EOL>(r"<STR_LIT>", MemoryMapGetAllHandler, dict(state_map=state_map)),<EOL>(r"<STR_LIT>", MemoryMapHandler, dict(state_map=state_map)),<EOL>(r"<STR_LIT>", StateResultHandler, dict(result_map=state_result_map)),<EOL>])<EOL>if len(sys.argv) == <NUM_LIT:1>:<EOL><INDENT>logging.error("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>try:<EOL><INDENT>port = int(sys.argv[<NUM_LIT:1>])<EOL><DEDENT>except Exception as e:<EOL><INDENT>logging.error("<STR_LIT>", e)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>logging.info("<STR_LIT>" + str(port))<EOL>application.listen(port)<EOL>tornado.ioloop.IOLoop.instance().start()<EOL>

Runs a tornado http server that listens for any
integration test json result get/post requests


topology = protoTopology.Topology()<EOL>topology.id = MockProto.topology_id<EOL>topology.name = MockProto.topology_name<EOL>stream1 = protoTopology.StreamId()<EOL>stream1.id = "<STR_LIT>"<EOL>stream1.component_name = "<STR_LIT>"<EOL>spout = self.create_mock_spout("<STR_LIT>", [stream1], spout_parallelism)<EOL>topology.spouts.extend([spout])<EOL>bolt = self.create_mock_bolt("<STR_LIT>", [stream1], [], bolt_parallelism)<EOL>topology.bolts.extend([bolt])<EOL>return topology<EOL>

Simple topology contains one spout and one bolt.


topology = protoTopology.Topology()<EOL>topology.id = "<STR_LIT>"<EOL>topology.name = "<STR_LIT>"<EOL>stream1 = protoTopology.StreamId()<EOL>stream1.id = "<STR_LIT>"<EOL>stream1.component_name = "<STR_LIT>"<EOL>stream2 = protoTopology.StreamId()<EOL>stream2.id = "<STR_LIT>"<EOL>stream2.component_name = "<STR_LIT>"<EOL>stream3 = protoTopology.StreamId()<EOL>stream3.id = "<STR_LIT>"<EOL>stream3.component_name = "<STR_LIT>"<EOL>stream4 = protoTopology.StreamId()<EOL>stream4.id = "<STR_LIT>"<EOL>stream4.component_name = "<STR_LIT>"<EOL>spout1 = self.create_mock_spout("<STR_LIT>",<EOL>[stream1, stream2],<EOL>spout_parallelism)<EOL>topology.spouts.extend([spout1])<EOL>bolt1 = self.create_mock_bolt("<STR_LIT>",<EOL>[stream1],<EOL>[stream3],<EOL>bolt1_parallelism)<EOL>bolt2 = self.create_mock_bolt("<STR_LIT>",<EOL>[stream2],<EOL>[stream4],<EOL>bolt2_parallelism)<EOL>bolt3 = self.create_mock_bolt("<STR_LIT>",<EOL>[stream3, stream4],<EOL>[],<EOL>bolt3_parallelism)<EOL>topology.bolts.extend([bolt1, bolt2, bolt3])<EOL>return topology<EOL>

Medium topology is a three stage topology
with one spout, two mid stage bolts, and one
last stage bolt.
S -str1-> B1 -str3-> B3
S -str2-> B2 -str4-> B3


self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])<EOL>if EXTRA_LINKS_KEY in self.configs:<EOL><INDENT>for extra_link in self.configs[EXTRA_LINKS_KEY]:<EOL><INDENT>self.extra_links.append(self.validate_extra_link(extra_link))<EOL><DEDENT><DEDENT>


if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:<EOL><INDENT>raise Exception("<STR_LIT>" +<EOL>"<STR_LIT>")<EOL><DEDENT>self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY])<EOL>return extra_link<EOL>


<EOL>valid_parameters = {<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:user>",<EOL>}<EOL>dummy_formatted_url = url_format<EOL>for key, value in valid_parameters.items():<EOL><INDENT>dummy_formatted_url = dummy_formatted_url.replace(key, value)<EOL><DEDENT>if '<STR_LIT:$>' in dummy_formatted_url:<EOL><INDENT>raise Exception("<STR_LIT>" % (url_format))<EOL><DEDENT>return url_format<EOL>


def get_formatted_url(self, execution_state, formatter):

<EOL>valid_parameters = {<EOL>"<STR_LIT>": execution_state["<STR_LIT>"],<EOL>"<STR_LIT>": execution_state["<STR_LIT>"],<EOL>"<STR_LIT>": execution_state["<STR_LIT>"],<EOL>"<STR_LIT>": execution_state["<STR_LIT>"],<EOL>"<STR_LIT>": execution_state["<STR_LIT>"],<EOL>}<EOL>formatted_url = formatter<EOL>for key, value in valid_parameters.items():<EOL><INDENT>formatted_url = formatted_url.replace(key, value)<EOL><DEDENT>return formatted_url<EOL>

@param execution_state: The python dict representing JSON execution_state
@return Formatted viz url


RETRY_COUNT = <NUM_LIT:5><EOL>for _ in range(RETRY_COUNT):<EOL><INDENT>uid = uuid.uuid4()<EOL>if uid not in self.watches:<EOL><INDENT>Log.info("<STR_LIT>" + str(uid))<EOL>try:<EOL><INDENT>callback(self)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>" + str(e))<EOL>Log.debug(traceback.format_exc())<EOL>return None<EOL><DEDENT>self.watches[uid] = callback<EOL>return uid<EOL><DEDENT><DEDENT>return None<EOL>

Returns the UUID with which the watch is
registered. This UUID can be used to unregister
the watch.
Returns None if watch could not be registered.

The argument 'callback' must be a function that takes
exactly one argument, the topology on which
the watch was triggered.
Note that the watch will be unregistered in case
it raises any Exception the first time.

This callback is also called at the time
of registration.


<EOL>Log.info("<STR_LIT>" + str(uid))<EOL>self.watches.pop(uid, None)<EOL>

Unregister the watch with the given UUID.


to_remove = []<EOL>for uid, callback in self.watches.items():<EOL><INDENT>try:<EOL><INDENT>callback(self)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>" + str(e))<EOL>Log.debug(traceback.format_exc())<EOL>to_remove.append(uid)<EOL><DEDENT><DEDENT>for uid in to_remove:<EOL><INDENT>self.unregister_watch(uid)<EOL><DEDENT>

Call all the callbacks.
If any callback raises an Exception,
unregister the corresponding watch.


if not physical_plan:<EOL><INDENT>self.physical_plan = None<EOL>self.id = None<EOL><DEDENT>else:<EOL><INDENT>self.physical_plan = physical_plan<EOL>self.id = physical_plan.topology.id<EOL><DEDENT>self.trigger_watches()<EOL>


if not packing_plan:<EOL><INDENT>self.packing_plan = None<EOL>self.id = None<EOL><DEDENT>else:<EOL><INDENT>self.packing_plan = packing_plan<EOL>self.id = packing_plan.id<EOL><DEDENT>self.trigger_watches()<EOL>


return (execution_state.cluster, execution_state.environ)<EOL>

Helper function to extract dc and environ from execution_state.
Returns a tuple (cluster, environ).


if not execution_state:<EOL><INDENT>self.execution_state = None<EOL>self.cluster = None<EOL>self.environ = None<EOL><DEDENT>else:<EOL><INDENT>self.execution_state = execution_state<EOL>cluster, environ = self.get_execution_state_dc_environ(execution_state)<EOL>self.cluster = cluster<EOL>self.environ = environ<EOL>self.zone = cluster<EOL><DEDENT>self.trigger_watches()<EOL>


self.tmaster = tmaster<EOL>self.trigger_watches()<EOL>


self.scheduler_location = scheduler_location<EOL>self.trigger_watches()<EOL>


num = <NUM_LIT:0><EOL>components = self.spouts() + self.bolts()<EOL>for component in components:<EOL><INDENT>config = component.comp.config<EOL>for kvs in config.kvs:<EOL><INDENT>if kvs.key == api_constants.TOPOLOGY_COMPONENT_PARALLELISM:<EOL><INDENT>num += int(kvs.value)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>return num<EOL>


if self.physical_plan:<EOL><INDENT>return list(self.physical_plan.topology.spouts)<EOL><DEDENT>return []<EOL>

Returns a list of Spout (proto) messages


return map(lambda component: component.comp.name, self.spouts())<EOL>

Returns a list of names of all the spouts


if self.physical_plan:<EOL><INDENT>return list(self.physical_plan.topology.bolts)<EOL><DEDENT>return []<EOL>


return map(lambda component: component.comp.name, self.bolts())<EOL>

Returns a list of names of all the bolts


if self.physical_plan:<EOL><INDENT>stmgrs = list(self.physical_plan.stmgrs)<EOL>return map(lambda s: s.host_name, stmgrs)<EOL><DEDENT>return []<EOL>

Get all the machines that this topology is running on.
These are the hosts of all the stmgrs.


status = None<EOL>if self.physical_plan and self.physical_plan.topology:<EOL><INDENT>status = self.physical_plan.topology.state<EOL><DEDENT>if status == <NUM_LIT:1>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>elif status == <NUM_LIT:2>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>elif status == <NUM_LIT:3>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>

Get the current state of this topology.
The state values are from the topology.proto
RUNNING = 1, PAUSED = 2, KILLED = 3
if the state is None "Unknown" is returned.


<EOL>if not tmaster or not tmaster.host or not tmaster.stats_port:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>host = tmaster.host<EOL>port = tmaster.stats_port<EOL>metricRequest = tmaster_pb2.MetricRequest()<EOL>metricRequest.component_name = component_name<EOL>if len(instances) > <NUM_LIT:0>:<EOL><INDENT>for instance in instances:<EOL><INDENT>metricRequest.instance_id.append(instance)<EOL><DEDENT><DEDENT>for metricName in metric_names:<EOL><INDENT>metricRequest.metric.append(metricName)<EOL><DEDENT>metricRequest.explicit_interval.start = start_time<EOL>metricRequest.explicit_interval.end = end_time<EOL>metricRequest.minutely = True<EOL>metricRequestString = metricRequest.SerializeToString()<EOL>url = "<STR_LIT>".format(host, port)<EOL>request = tornado.httpclient.HTTPRequest(url,<EOL>body=metricRequestString,<EOL>method='<STR_LIT:POST>',<EOL>request_timeout=<NUM_LIT:5>)<EOL>Log.debug("<STR_LIT>")<EOL>Log.debug("<STR_LIT>" + url)<EOL>try:<EOL><INDENT>client = tornado.httpclient.AsyncHTTPClient()<EOL>result = yield client.fetch(request)<EOL>Log.debug("<STR_LIT>")<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>responseCode = result.code<EOL>if responseCode >= <NUM_LIT>:<EOL><INDENT>message = "<STR_LIT>" + responseCode<EOL>Log.error(message)<EOL>raise Exception(message)<EOL><DEDENT>metricResponse = tmaster_pb2.MetricResponse()<EOL>metricResponse.ParseFromString(result.body)<EOL>if metricResponse.status.status == common_pb2.NOTOK:<EOL><INDENT>if metricResponse.status.HasField("<STR_LIT:message>"):<EOL><INDENT>Log.warn("<STR_LIT>", metricResponse.status.message)<EOL><DEDENT><DEDENT>ret = {}<EOL>ret["<STR_LIT>"] = start_time<EOL>ret["<STR_LIT>"] = end_time<EOL>ret["<STR_LIT>"] = component_name<EOL>ret["<STR_LIT>"] = {}<EOL>for metric in metricResponse.metric:<EOL><INDENT>instance = metric.instance_id<EOL>for im in metric.metric:<EOL><INDENT>metricname = im.name<EOL>if metricname not in ret["<STR_LIT>"]:<EOL><INDENT>ret["<STR_LIT>"][metricname] = {}<EOL><DEDENT>if instance not in ret["<STR_LIT>"][metricname]:<EOL><INDENT>ret["<STR_LIT>"][metricname][instance] = {}<EOL><DEDENT>for interval_value in im.interval_values:<EOL><INDENT>ret["<STR_LIT>"][metricname][instance][interval_value.interval.start] = interval_value.value<EOL><DEDENT><DEDENT><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get the specified metrics for the given component name of this topology.
Returns the following dict on success:
{
  "timeline": {
    <metricname>: {
      <instance>: {
        <start_time> : <numeric value>,
        <start_time> : <numeric value>,
        ...
      }
      ...
    }, ...
  },
  "starttime": <numeric value>,
  "endtime": <numeric value>,
  "component": "..."
}

Returns the following dict on failure:
{
  "message": "..."
}


Log.debug("<STR_LIT:U+0020>" * (ident * <NUM_LIT:2>) + str(message))<EOL>


Log.error("<STR_LIT:U+0020>" * (ident * <NUM_LIT:2>) + str(message))<EOL>


marshaller = JavaObjectUnmarshaller(file_object)<EOL>marshaller.add_transformer(DefaultObjectTransformer())<EOL>return marshaller.readObject()<EOL>

Deserializes Java primitive data and objects serialized by ObjectOutputStream
from a file-like object.


f = StringIO.StringIO(string)<EOL>marshaller = JavaObjectUnmarshaller(f)<EOL>marshaller.add_transformer(DefaultObjectTransformer())<EOL>return marshaller.readObject()<EOL>

Deserializes Java objects and primitive data serialized by ObjectOutputStream
from a string.


marshaller = JavaObjectMarshaller()<EOL>return marshaller.dump(obj)<EOL>

Serializes Java primitive data and objects unmarshaled by load(s) before into string.


return self.classdesc<EOL>


return self.__repr__()<EOL>


name = "<STR_LIT>"<EOL>if self.classdesc:<EOL><INDENT>name = self.classdesc.name<EOL><DEDENT>return "<STR_LIT>" % name<EOL>


new_object.classdesc = self.classdesc<EOL>for name in self.classdesc.fields_names:<EOL><INDENT>new_object.__setattr__(name, getattr(self, name))<EOL><DEDENT>


try:<EOL><INDENT>_, res = self._read_and_exec_opcode(ident=<NUM_LIT:0>)<EOL>position_bak = self.object_stream.tell()<EOL>the_rest = self.object_stream.read()<EOL>if len(the_rest):<EOL><INDENT>log_error("<STR_LIT>" % len(the_rest))<EOL>log_debug(self._create_hexdump(the_rest))<EOL><DEDENT>else:<EOL><INDENT>log_debug("<STR_LIT>")<EOL><DEDENT>self.object_stream.seek(position_bak)<EOL>return res<EOL><DEDENT>except Exception:<EOL><INDENT>self._oops_dump_state()<EOL>raise<EOL><DEDENT>


self.object_transformers.append(transformer)<EOL>


def do_classdesc(self, parent=None, ident=<NUM_LIT:0>):

<EOL>clazz = JavaClass()<EOL>log_debug("<STR_LIT>", ident)<EOL>ba = self._readString()<EOL>clazz.name = ba<EOL>log_debug("<STR_LIT>" % ba, ident)<EOL>(serialVersionUID, newHandle, classDescFlags) = self._readStruct("<STR_LIT>")<EOL>clazz.serialVersionUID = serialVersionUID<EOL>clazz.flags = classDescFlags<EOL>self._add_reference(clazz)<EOL>log_debug("<STR_LIT>" % (serialVersionUID, newHandle, classDescFlags), ident)<EOL>(length, ) = self._readStruct("<STR_LIT>")<EOL>log_debug("<STR_LIT>" % length, ident)<EOL>clazz.fields_names = []<EOL>clazz.fields_types = []<EOL>for _ in range(length):<EOL><INDENT>(typecode, ) = self._readStruct("<STR_LIT>")<EOL>field_name = self._readString()<EOL>field_type = None<EOL>field_type = self._convert_char_to_type(typecode)<EOL>if field_type == self.TYPE_ARRAY:<EOL><INDENT>_, field_type = self._read_and_exec_opcode(<EOL>ident=ident+<NUM_LIT:1>, expect=[self.TC_STRING, self.TC_REFERENCE])<EOL>assert isinstance(field_type, str)<EOL>


self.tracker = tracker<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>environ = self.get_argument_environ()<EOL>role = self.get_argument_role()<EOL>topology_name = self.get_argument_topology()<EOL>component = self.get_argument_component()<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topology_name)<EOL>instances = self.get_arguments(constants.PARAM_INSTANCE)<EOL>exceptions_summary = yield tornado.gen.Task(self.getComponentExceptionSummary,<EOL>topology.tmaster, component, instances)<EOL>self.write_success_response(exceptions_summary)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def getComponentExceptionSummary(self, tmaster, component_name, instances=[], callback=None):<DEDENT>

if not tmaster or not tmaster.host or not tmaster.stats_port:<EOL><INDENT>return<EOL><DEDENT>exception_request = tmaster_pb2.ExceptionLogRequest()<EOL>exception_request.component_name = component_name<EOL>if len(instances) > <NUM_LIT:0>:<EOL><INDENT>exception_request.instances.extend(instances)<EOL><DEDENT>request_str = exception_request.SerializeToString()<EOL>port = str(tmaster.stats_port)<EOL>host = tmaster.host<EOL>url = "<STR_LIT>".format(host, port)<EOL>Log.debug("<STR_LIT>")<EOL>request = tornado.httpclient.HTTPRequest(url,<EOL>body=request_str,<EOL>method='<STR_LIT:POST>',<EOL>request_timeout=<NUM_LIT:5>)<EOL>Log.debug('<STR_LIT>', url)<EOL>try:<EOL><INDENT>client = tornado.httpclient.AsyncHTTPClient()<EOL>result = yield client.fetch(request)<EOL>Log.debug("<STR_LIT>")<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>responseCode = result.code<EOL>if responseCode >= <NUM_LIT>:<EOL><INDENT>message = "<STR_LIT>" + responseCode<EOL>Log.error(message)<EOL>raise tornado.gen.Return({<EOL>"<STR_LIT:message>": message<EOL>})<EOL><DEDENT>exception_response = tmaster_pb2.ExceptionLogResponse()<EOL>exception_response.ParseFromString(result.body)<EOL>if exception_response.status.status == common_pb2.NOTOK:<EOL><INDENT>if exception_response.status.HasField("<STR_LIT:message>"):<EOL><INDENT>raise tornado.gen.Return({<EOL>"<STR_LIT:message>": exception_response.status.message<EOL>})<EOL><DEDENT><DEDENT>ret = []<EOL>for exception_log in exception_response.exceptions:<EOL><INDENT>ret.append({'<STR_LIT>': exception_log.stacktrace,<EOL>'<STR_LIT>': exception_log.lasttime,<EOL>'<STR_LIT>': exception_log.firsttime,<EOL>'<STR_LIT:count>': str(exception_log.count)})<EOL><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get the summary of exceptions for component_name and list of instances.
Empty instance list will fetch all exceptions.


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>instance = self.get_argument_instance()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>ret = yield self.getInstanceJstack(topology_info, instance)<EOL>self.write_success_response(ret)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def getInstanceJstack(self, topology_info, instance_id):<DEDENT>

pid_response = yield getInstancePid(topology_info, instance_id)<EOL>try:<EOL><INDENT>http_client = tornado.httpclient.AsyncHTTPClient()<EOL>pid_json = json.loads(pid_response)<EOL>pid = pid_json['<STR_LIT>'].strip()<EOL>if pid == '<STR_LIT>':<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>endpoint = utils.make_shell_endpoint(topology_info, instance_id)<EOL>url = "<STR_LIT>" % (endpoint, pid)<EOL>response = yield http_client.fetch(url)<EOL>Log.debug("<STR_LIT>", url)<EOL>raise tornado.gen.Return(response.body)<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>

Fetches Instance jstack from heron-shell.


self.set_header("<STR_LIT>", "<STR_LIT:*>")<EOL>

Allow any domain to make queries to tracker.


self.basehandler_starttime = time.time()<EOL>

Used for timing. Sets the basehandler_starttime to current time, and
is used when writing the response back.
Subclasses of BaseHandler must never use self.write, but instead use
self.write_error or self.write_result methods to correctly include
the timing.


response = self.make_success_response(result)<EOL>now = time.time()<EOL>spent = now - self.basehandler_starttime<EOL>response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent<EOL>self.write_json_response(response)<EOL>

Result may be a python dictionary, array or a primitive type
that can be converted to JSON for writing back the result.


self.set_status(<NUM_LIT>)<EOL>response = self.make_error_response(str(message))<EOL>now = time.time()<EOL>spent = now - self.basehandler_starttime<EOL>response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent<EOL>self.write_json_response(response)<EOL>

Writes the message as part of the response and sets 404 status.


self.write(tornado.escape.json_encode(response))<EOL>self.set_header("<STR_LIT:Content-Type>", "<STR_LIT:application/json>")<EOL>


response = {<EOL>constants.RESPONSE_KEY_STATUS: status,<EOL>constants.RESPONSE_KEY_VERSION: constants.API_VERSION,<EOL>constants.RESPONSE_KEY_EXECUTION_TIME: <NUM_LIT:0>,<EOL>constants.RESPONSE_KEY_MESSAGE: "<STR_LIT>",<EOL>}<EOL>return response<EOL>

Makes the base dict for the response.
The status is the string value for
the key "status" of the response. This
should be "success" or "failure".


response = self.make_response(constants.RESPONSE_STATUS_SUCCESS)<EOL>response[constants.RESPONSE_KEY_RESULT] = result<EOL>return response<EOL>

Makes the python dict corresponding to the
JSON that needs to be sent for a successful
response. Result is the actual payload
that gets sent.


response = self.make_response(constants.RESPONSE_STATUS_FAILURE)<EOL>response[constants.RESPONSE_KEY_MESSAGE] = message<EOL>return response<EOL>

Makes the python dict corresponding to the
JSON that needs to be sent for a failed
response. Message is the message that is
sent as the reason for failure.


try:<EOL><INDENT>return self.get_argument(constants.PARAM_CLUSTER)<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get request argument.
Raises exception if argument is missing.
Returns the cluster argument.


try:<EOL><INDENT>return self.get_argument(constants.PARAM_ROLE, default=None)<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get request argument.
Raises exception if argument is missing.
Returns the role argument.


try:<EOL><INDENT>return self.get_argument(constants.PARAM_ENVIRON)<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get request argument.
Raises exception if argument is missing.
Returns the environ argument.


try:<EOL><INDENT>topology = self.get_argument(constants.PARAM_TOPOLOGY)<EOL>return topology<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get topology argument.
Raises exception if argument is missing.
Returns the topology argument.


try:<EOL><INDENT>component = self.get_argument(constants.PARAM_COMPONENT)<EOL>return component<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get component argument.
Raises exception if argument is missing.
Returns the component argument.


try:<EOL><INDENT>instance = self.get_argument(constants.PARAM_INSTANCE)<EOL>return instance<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get instance argument.
Raises exception if argument is missing.
Returns the instance argument.


try:<EOL><INDENT>starttime = self.get_argument(constants.PARAM_STARTTIME)<EOL>return starttime<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get starttime argument.
Raises exception if argument is missing.
Returns the starttime argument.


try:<EOL><INDENT>endtime = self.get_argument(constants.PARAM_ENDTIME)<EOL>return endtime<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get endtime argument.
Raises exception if argument is missing.
Returns the endtime argument.


try:<EOL><INDENT>query = self.get_argument(constants.PARAM_QUERY)<EOL>return query<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get query argument.
Raises exception if argument is missing.
Returns the query argument.


try:<EOL><INDENT>offset = self.get_argument(constants.PARAM_OFFSET)<EOL>return offset<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get offset argument.
Raises exception if argument is missing.
Returns the offset argument.


try:<EOL><INDENT>length = self.get_argument(constants.PARAM_LENGTH)<EOL>return length<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get length argument.
Raises exception if argument is missing.
Returns the length argument.


try:<EOL><INDENT>metricnames = self.get_arguments(constants.PARAM_METRICNAME)<EOL>if not metricnames:<EOL><INDENT>raise tornado.web.MissingArgumentError(constants.PARAM_METRICNAME)<EOL><DEDENT>return metricnames<EOL><DEDENT>except tornado.web.MissingArgumentError as e:<EOL><INDENT>raise Exception(e.log_message)<EOL><DEDENT>

Helper function to get metricname arguments.
Notice that it is get_argument"s" variation, which means that this can be repeated.
Raises exception if argument is missing.
Returns a list of metricname arguments


def validateInterval(self, startTime, endTime):

start = int(startTime)<EOL>end = int(endTime)<EOL>if start > end:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>

Helper function to validate interval.
An interval is valid if starttime and endtime are integrals,
and starttime is less than the endtime.
Raises exception if interval is not valid.


self.tracker = tracker<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>container = self.get_argument(constants.PARAM_CONTAINER)<EOL>path = self.get_argument(constants.PARAM_PATH)<EOL>offset = self.get_argument_offset()<EOL>length = self.get_argument_length()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>stmgr_id = "<STR_LIT>" + container<EOL>stmgr = topology_info["<STR_LIT>"]["<STR_LIT>"][stmgr_id]<EOL>host = stmgr["<STR_LIT:host>"]<EOL>shell_port = stmgr["<STR_LIT>"]<EOL>file_data_url = "<STR_LIT>" %(host, shell_port, path, offset, length)<EOL>http_client = tornado.httpclient.AsyncHTTPClient()<EOL>response = yield http_client.fetch(file_data_url)<EOL>self.write_success_response(json.loads(response.body))<EOL>self.finish()<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>container = self.get_argument(constants.PARAM_CONTAINER)<EOL>path = self.get_argument(constants.PARAM_PATH, default="<STR_LIT:.>")<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>stmgr_id = "<STR_LIT>" + str(container)<EOL>stmgr = topology_info["<STR_LIT>"]["<STR_LIT>"][stmgr_id]<EOL>host = stmgr["<STR_LIT:host>"]<EOL>shell_port = stmgr["<STR_LIT>"]<EOL>filestats_url = utils.make_shell_filestats_url(host, shell_port, path)<EOL>http_client = tornado.httpclient.AsyncHTTPClient()<EOL>response = yield http_client.fetch(filestats_url)<EOL>self.write_success_response(json.loads(response.body))<EOL>self.finish()<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>config = topology_info["<STR_LIT>"]["<STR_LIT>"]<EOL>self.write_success_response(config)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def getStmgrsRegSummary(self, tmaster, callback=None):<DEDENT>

if not tmaster or not tmaster.host or not tmaster.stats_port:<EOL><INDENT>return<EOL><DEDENT>reg_request = tmaster_pb2.StmgrsRegistrationSummaryRequest()<EOL>request_str = reg_request.SerializeToString()<EOL>port = str(tmaster.stats_port)<EOL>host = tmaster.host<EOL>url = "<STR_LIT>".format(host, port)<EOL>request = tornado.httpclient.HTTPRequest(url,<EOL>body=request_str,<EOL>method='<STR_LIT:POST>',<EOL>request_timeout=<NUM_LIT:5>)<EOL>Log.debug('<STR_LIT>', url)<EOL>try:<EOL><INDENT>client = tornado.httpclient.AsyncHTTPClient()<EOL>result = yield client.fetch(request)<EOL>Log.debug("<STR_LIT>")<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>responseCode = result.code<EOL>if responseCode >= <NUM_LIT>:<EOL><INDENT>message = "<STR_LIT>" + responseCode<EOL>Log.error(message)<EOL>raise tornado.gen.Return({<EOL>"<STR_LIT:message>": message<EOL>})<EOL><DEDENT>reg_response = tmaster_pb2.StmgrsRegistrationSummaryResponse()<EOL>reg_response.ParseFromString(result.body)<EOL>ret = {}<EOL>for stmgr in reg_response.registered_stmgrs:<EOL><INDENT>ret[stmgr] = True<EOL><DEDENT>for stmgr in reg_response.absent_stmgrs:<EOL><INDENT>ret[stmgr] = False<EOL><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get summary of stream managers registration summary


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>runtime_state = topology_info["<STR_LIT>"]<EOL>runtime_state["<STR_LIT>"] = topology_info["<STR_LIT>"]["<STR_LIT>"]<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topology_name)<EOL>reg_summary = yield tornado.gen.Task(self.getStmgrsRegSummary, topology.tmaster)<EOL>for stmgr, reg in reg_summary.items():<EOL><INDENT>runtime_state["<STR_LIT>"].setdefault(stmgr, {})["<STR_LIT>"] = reg<EOL><DEDENT>self.write_success_response(runtime_state)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>component = self.get_argument_component()<EOL>metric_names = self.get_required_arguments_metricnames()<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topology_name)<EOL>interval = int(self.get_argument(constants.PARAM_INTERVAL, default=-<NUM_LIT:1>))<EOL>instances = self.get_arguments(constants.PARAM_INSTANCE)<EOL>metrics = yield tornado.gen.Task(<EOL>self.getComponentMetrics,<EOL>topology.tmaster, component, metric_names, instances, interval)<EOL>self.write_success_response(metrics)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


if not tmaster or not tmaster.host or not tmaster.stats_port:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>host = tmaster.host<EOL>port = tmaster.stats_port<EOL>metricRequest = tmaster_pb2.MetricRequest()<EOL>metricRequest.component_name = componentName<EOL>if len(instances) > <NUM_LIT:0>:<EOL><INDENT>for instance in instances:<EOL><INDENT>metricRequest.instance_id.append(instance)<EOL><DEDENT><DEDENT>for metricName in metricNames:<EOL><INDENT>metricRequest.metric.append(metricName)<EOL><DEDENT>metricRequest.interval = interval<EOL>metricRequestString = metricRequest.SerializeToString()<EOL>url = "<STR_LIT>".format(host, port)<EOL>request = tornado.httpclient.HTTPRequest(url,<EOL>body=metricRequestString,<EOL>method='<STR_LIT:POST>',<EOL>request_timeout=<NUM_LIT:5>)<EOL>Log.debug("<STR_LIT>")<EOL>Log.debug("<STR_LIT>" + url)<EOL>try:<EOL><INDENT>client = tornado.httpclient.AsyncHTTPClient()<EOL>result = yield client.fetch(request)<EOL>Log.debug("<STR_LIT>")<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>responseCode = result.code<EOL>if responseCode >= <NUM_LIT>:<EOL><INDENT>message = "<STR_LIT>" + responseCode<EOL>Log.error(message)<EOL>raise Exception(message)<EOL><DEDENT>metricResponse = tmaster_pb2.MetricResponse()<EOL>metricResponse.ParseFromString(result.body)<EOL>if metricResponse.status.status == common_pb2.NOTOK:<EOL><INDENT>if metricResponse.status.HasField("<STR_LIT:message>"):<EOL><INDENT>Log.warn("<STR_LIT>", metricResponse.status.message)<EOL><DEDENT><DEDENT>ret = {}<EOL>ret["<STR_LIT>"] = metricResponse.interval<EOL>ret["<STR_LIT>"] = componentName<EOL>ret["<STR_LIT>"] = {}<EOL>for metric in metricResponse.metric:<EOL><INDENT>instance = metric.instance_id<EOL>for im in metric.metric:<EOL><INDENT>metricname = im.name<EOL>value = im.value<EOL>if metricname not in ret["<STR_LIT>"]:<EOL><INDENT>ret["<STR_LIT>"][metricname] = {}<EOL><DEDENT>ret["<STR_LIT>"][metricname][instance] = value<EOL><DEDENT><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get the specified metrics for the given component name of this topology.
Returns the following dict on success:
{
  "metrics": {
    <metricname>: {
      <instance>: <numeric value>,
      <instance>: <numeric value>,
      ...
    }, ...
  },
  "interval": <numeric value>,
  "component": "..."
}

Raises exception on failure.


clusters = [statemgr.name for statemgr in self.tracker.state_managers]<EOL>self.write_success_response(clusters)<EOL>


clusters = self.get_arguments(constants.PARAM_CLUSTER)<EOL>environs = self.get_arguments(constants.PARAM_ENVIRON)<EOL>topology_names = self.get_arguments(constants.PARAM_TOPOLOGY)<EOL>ret = {}<EOL>if len(topology_names) > <NUM_LIT:1>:<EOL><INDENT>if not clusters:<EOL><INDENT>message = "<STR_LIT>" + constants.PARAM_CLUSTER<EOL>self.write_error_response(message)<EOL>return<EOL><DEDENT>if not environs:<EOL><INDENT>message = "<STR_LIT>" + constants.PARAM_ENVIRON<EOL>self.write_error_response(message)<EOL>return<EOL><DEDENT><DEDENT>ret = {}<EOL>topologies = self.tracker.topologies<EOL>for topology in topologies:<EOL><INDENT>cluster = topology.cluster<EOL>environ = topology.environ<EOL>topology_name = topology.name<EOL>if not cluster or not environ:<EOL><INDENT>continue<EOL><DEDENT>if clusters and cluster not in clusters:<EOL><INDENT>continue<EOL><DEDENT>if environs and environ not in environs:<EOL><INDENT>continue<EOL><DEDENT>if topology_names and topology_name not in topology_names:<EOL><INDENT>continue<EOL><DEDENT>if cluster not in ret:<EOL><INDENT>ret[cluster] = {}<EOL><DEDENT>if environ not in ret[cluster]:<EOL><INDENT>ret[cluster][environ] = {}<EOL><DEDENT>ret[cluster][environ][topology_name] = topology.get_machines()<EOL><DEDENT>self.write_success_response(ret)<EOL>


self.write_error_response("<STR_LIT>" + url)<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>component = self.get_argument_component()<EOL>metric_names = self.get_required_arguments_metricnames()<EOL>start_time = self.get_argument_starttime()<EOL>end_time = self.get_argument_endtime()<EOL>self.validateInterval(start_time, end_time)<EOL>instances = self.get_arguments(constants.PARAM_INSTANCE)<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topology_name)<EOL>metrics = yield tornado.gen.Task(metricstimeline.getMetricsTimeline,<EOL>topology.tmaster, component, metric_names,<EOL>instances, int(start_time), int(end_time))<EOL>self.write_success_response(metrics)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>execution_state = topology_info["<STR_LIT>"]<EOL>self.write_success_response(execution_state)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>self.write_success_response(topology_info)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>packing_plan = topology_info["<STR_LIT>"]<EOL>self.write_success_response(packing_plan)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>instance = self.get_argument_instance()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>ret = yield self.runInstanceJmap(topology_info, instance)<EOL>self.write_success_response(ret)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def runInstanceJmap(self, topology_info, instance_id):<DEDENT>

pid_response = yield getInstancePid(topology_info, instance_id)<EOL>try:<EOL><INDENT>http_client = tornado.httpclient.AsyncHTTPClient()<EOL>pid_json = json.loads(pid_response)<EOL>pid = pid_json['<STR_LIT>'].strip()<EOL>if pid == '<STR_LIT>':<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>endpoint = utils.make_shell_endpoint(topology_info, instance_id)<EOL>url = "<STR_LIT>" % (endpoint, pid)<EOL>response = yield http_client.fetch(url)<EOL>Log.debug("<STR_LIT>", url)<EOL>raise tornado.gen.Return(response.body)<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>

Fetches Instance jstack from heron-shell.


self.tracker = tracker<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>environ = self.get_argument_environ()<EOL>role = self.get_argument_role()<EOL>topName = self.get_argument_topology()<EOL>component = self.get_argument_component()<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topName)<EOL>instances = self.get_arguments(constants.PARAM_INSTANCE)<EOL>exceptions_logs = yield tornado.gen.Task(self.getComponentException,<EOL>topology.tmaster, component, instances)<EOL>self.write_success_response(exceptions_logs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def getComponentException(self, tmaster, component_name, instances=[], callback=None):<DEDENT>

if not tmaster or not tmaster.host or not tmaster.stats_port:<EOL><INDENT>return<EOL><DEDENT>exception_request = tmaster_pb2.ExceptionLogRequest()<EOL>exception_request.component_name = component_name<EOL>if len(instances) > <NUM_LIT:0>:<EOL><INDENT>exception_request.instances.extend(instances)<EOL><DEDENT>request_str = exception_request.SerializeToString()<EOL>port = str(tmaster.stats_port)<EOL>host = tmaster.host<EOL>url = "<STR_LIT>".format(host, port)<EOL>request = tornado.httpclient.HTTPRequest(url,<EOL>body=request_str,<EOL>method='<STR_LIT:POST>',<EOL>request_timeout=<NUM_LIT:5>)<EOL>Log.debug('<STR_LIT>', url)<EOL>try:<EOL><INDENT>client = tornado.httpclient.AsyncHTTPClient()<EOL>result = yield client.fetch(request)<EOL>Log.debug("<STR_LIT>")<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>responseCode = result.code<EOL>if responseCode >= <NUM_LIT>:<EOL><INDENT>message = "<STR_LIT>" + responseCode<EOL>Log.error(message)<EOL>raise tornado.gen.Return({<EOL>"<STR_LIT:message>": message<EOL>})<EOL><DEDENT>exception_response = tmaster_pb2.ExceptionLogResponse()<EOL>exception_response.ParseFromString(result.body)<EOL>if exception_response.status.status == common_pb2.NOTOK:<EOL><INDENT>if exception_response.status.HasField("<STR_LIT:message>"):<EOL><INDENT>raise tornado.gen.Return({<EOL>"<STR_LIT:message>": exception_response.status.message<EOL>})<EOL><DEDENT><DEDENT>ret = []<EOL>for exception_log in exception_response.exceptions:<EOL><INDENT>ret.append({'<STR_LIT>': exception_log.hostname,<EOL>'<STR_LIT>': exception_log.instance_id,<EOL>'<STR_LIT>': exception_log.stacktrace,<EOL>'<STR_LIT>': exception_log.lasttime,<EOL>'<STR_LIT>': exception_log.firsttime,<EOL>'<STR_LIT:count>': str(exception_log.count),<EOL>'<STR_LIT>': exception_log.logging})<EOL><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get all (last 1000) exceptions for 'component_name' of the topology.
Returns an Array of exception logs on success.
Returns json with message on failure.


<EOL>clusters = self.get_arguments(constants.PARAM_CLUSTER)<EOL>environs = self.get_arguments(constants.PARAM_ENVIRON)<EOL>role = self.get_argument_role()<EOL>ret = {}<EOL>topologies = self.tracker.topologies<EOL>for topology in topologies:<EOL><INDENT>cluster = topology.cluster<EOL>environ = topology.environ<EOL>if not cluster or not environ:<EOL><INDENT>continue<EOL><DEDENT>if clusters and cluster not in clusters:<EOL><INDENT>continue<EOL><DEDENT>if environs and environ not in environs:<EOL><INDENT>continue<EOL><DEDENT>if cluster not in ret:<EOL><INDENT>ret[cluster] = {}<EOL><DEDENT>if environ not in ret[cluster]:<EOL><INDENT>ret[cluster][environ] = {}<EOL><DEDENT>try:<EOL><INDENT>topology_info = self.tracker.getTopologyInfo(topology.name, cluster, role, environ)<EOL>if topology_info and "<STR_LIT>" in topology_info:<EOL><INDENT>ret[cluster][environ][topology.name] = topology_info["<STR_LIT>"]<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>self.write_success_response(ret)<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>scheduler_location = topology_info["<STR_LIT>"]<EOL>self.write_success_response(scheduler_location)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>instance = self.get_argument_instance()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>ret = yield self.getInstanceMemoryHistogram(topology_info, instance)<EOL>self.write_success_response(ret)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def getInstanceMemoryHistogram(self, topology_info, instance_id):<DEDENT>

Fetches Instance top memory item as histogram.


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>metadata = topology_info["<STR_LIT>"]<EOL>self.write_success_response(metadata)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


<EOL>clusters = self.get_arguments(constants.PARAM_CLUSTER)<EOL>environs = self.get_arguments(constants.PARAM_ENVIRON)<EOL>role = self.get_argument_role()<EOL>ret = {}<EOL>topologies = self.tracker.topologies<EOL>for topology in topologies:<EOL><INDENT>cluster = topology.cluster<EOL>environ = topology.environ<EOL>execution_state = topology.execution_state<EOL>if not cluster or not execution_state or not environ:<EOL><INDENT>continue<EOL><DEDENT>topo_role = execution_state.role<EOL>if not topo_role:<EOL><INDENT>continue<EOL><DEDENT>if clusters and cluster not in clusters:<EOL><INDENT>continue<EOL><DEDENT>if environs and environ not in environs:<EOL><INDENT>continue<EOL><DEDENT>if role and role != topo_role:<EOL><INDENT>continue<EOL><DEDENT>if cluster not in ret:<EOL><INDENT>ret[cluster] = {}<EOL><DEDENT>if topo_role not in ret[cluster]:<EOL><INDENT>ret[cluster][topo_role] = {}<EOL><DEDENT>if environ not in ret[cluster][topo_role]:<EOL><INDENT>ret[cluster][topo_role][environ] = []<EOL><DEDENT>ret[cluster][topo_role][environ].append(topology.name)<EOL><DEDENT>self.write_success_response(ret)<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topology_name)<EOL>start_time = self.get_argument_starttime()<EOL>end_time = self.get_argument_endtime()<EOL>self.validateInterval(start_time, end_time)<EOL>query = self.get_argument_query()<EOL>metrics = yield tornado.gen.Task(self.executeMetricsQuery,<EOL>topology.tmaster, query, int(start_time), int(end_time))<EOL>self.write_success_response(metrics)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def executeMetricsQuery(self, tmaster, queryString, start_time, end_time, callback=None):<DEDENT>

query = Query(self.tracker)<EOL>metrics = yield query.execute_query(tmaster, queryString, start_time, end_time)<EOL>ret = {}<EOL>ret["<STR_LIT>"] = start_time<EOL>ret["<STR_LIT>"] = end_time<EOL>ret["<STR_LIT>"] = []<EOL>for metric in metrics:<EOL><INDENT>tl = {<EOL>"<STR_LIT:data>": metric.timeline<EOL>}<EOL>if metric.instance:<EOL><INDENT>tl["<STR_LIT>"] = metric.instance<EOL><DEDENT>ret["<STR_LIT>"].append(tl)<EOL><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get the specified metrics for the given query in this topology.
Returns the following dict on success:
{
  "timeline": [{
    "instance": <instance>,
    "data": {
      <start_time> : <numeric value>,
      <start_time> : <numeric value>,
      ...
    }
  }, {
    ...
  }, ...
  "starttime": <numeric value>,
  "endtime": <numeric value>,
},

Returns the following dict on failure:
{
  "message": "..."
}


try:<EOL><INDENT>http_client = tornado.httpclient.AsyncHTTPClient()<EOL>endpoint = utils.make_shell_endpoint(topology_info, instance_id)<EOL>url = "<STR_LIT>" % (endpoint, instance_id)<EOL>Log.debug("<STR_LIT>", url)<EOL>response = yield http_client.fetch(url)<EOL>raise tornado.gen.Return(response.body)<EOL><DEDENT>except tornado.httpclient.HTTPError as e:<EOL><INDENT>raise Exception(str(e))<EOL><DEDENT>

This method is used by other modules, and so it
is not a part of the class.
Fetches Instance pid from heron-shell.


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>instance = self.get_argument_instance()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>result = yield getInstancePid(topology_info, instance)<EOL>self.write_success_response(result)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>physical_plan = topology_info["<STR_LIT>"]<EOL>self.write_success_response(physical_plan)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>return parser<EOL>


default_config_file = os.path.join(<EOL>utils.get_heron_tracker_conf_dir(), constants.DEFAULT_CONFIG_FILE)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + default_config_file + '<STR_LIT>',<EOL>default=default_config_file)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>'+ str(constants.DEFAULT_STATE_MANAGER_TYPE) + '<STR_LIT:)>',<EOL>choices=["<STR_LIT:file>", "<STR_LIT>"])<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>'+ str(constants.DEFAULT_STATE_MANAGER_NAME) + '<STR_LIT:)>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>'+ str(constants.DEFAULT_STATE_MANAGER_ROOTPATH) + '<STR_LIT:)>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>'+ str(constants.DEFAULT_STATE_MANAGER_TUNNELHOST) + '<STR_LIT:)>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>''<STR_LIT>' + str(constants.DEFAULT_STATE_MANAGER_HOSTPORT) + '<STR_LIT:)>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + str(constants.DEFAULT_PORT) + '<STR_LIT:)>',<EOL>type=int,<EOL>default=constants.DEFAULT_PORT)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT:store_true>')<EOL>return parser<EOL>


parser = argparse.ArgumentParser(<EOL>epilog='<STR_LIT>',<EOL>usage="<STR_LIT>",<EOL>add_help=False)<EOL>parser = add_titles(parser)<EOL>parser = add_arguments(parser)<EOL>ya_parser = argparse.ArgumentParser(<EOL>parents=[parser],<EOL>formatter_class=SubcommandHelpFormatter,<EOL>add_help=False)<EOL>subparsers = ya_parser.add_subparsers(<EOL>title="<STR_LIT>")<EOL>help_parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=False)<EOL>help_parser.set_defaults(help=True)<EOL>subparsers.add_parser(<EOL>'<STR_LIT:version>',<EOL>help='<STR_LIT>',<EOL>add_help=True)<EOL>return parser, ya_parser<EOL>


define("<STR_LIT:port>", default=port)<EOL>define("<STR_LIT>", default=config_file)<EOL>


<EOL>(parser, _) = create_parsers()<EOL>(args, remaining) = parser.parse_known_args()<EOL>if remaining == ['<STR_LIT>']:<EOL><INDENT>parser.print_help()<EOL>parser.exit()<EOL><DEDENT>elif remaining == ['<STR_LIT:version>']:<EOL><INDENT>common_config.print_build_info()<EOL>parser.exit()<EOL><DEDENT>elif remaining != []:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>namespace = vars(args)<EOL>log.set_logging_level(namespace)<EOL>define_options(namespace['<STR_LIT:port>'], namespace['<STR_LIT>'])<EOL>config = Config(create_tracker_config(namespace))<EOL>application = Application(config)<EOL>def signal_handler(signum, frame):<EOL><INDENT>print('<STR_LIT:\n>', end='<STR_LIT>')<EOL>application.stop()<EOL>tornado.ioloop.IOLoop.instance().stop()<EOL><DEDENT>signal.signal(signal.SIGINT, signal_handler)<EOL>signal.signal(signal.SIGTERM, signal_handler)<EOL>Log.info("<STR_LIT>", namespace['<STR_LIT:port>'])<EOL>if namespace["<STR_LIT>"]:<EOL><INDENT>Log.info("<STR_LIT>", namespace['<STR_LIT>'])<EOL><DEDENT>Log.info("<STR_LIT>" + str(config))<EOL>http_server = tornado.httpserver.HTTPServer(application)<EOL>http_server.listen(namespace['<STR_LIT:port>'])<EOL>tornado.ioloop.IOLoop.instance().start()<EOL>


@tornado.gen.coroutine<EOL><INDENT>def execute_query(self, tmaster, query_string, start, end):<DEDENT>

if not tmaster:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self.tmaster = tmaster<EOL>root = self.parse_query_string(query_string)<EOL>metrics = yield root.execute(self.tracker, self.tmaster, start, end)<EOL>raise tornado.gen.Return(metrics)<EOL>


if query[<NUM_LIT:0>] != '<STR_LIT:(>':<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>num_open_braces = <NUM_LIT:0><EOL>for i in range(len(query)):<EOL><INDENT>c = query[i]<EOL>if c == '<STR_LIT:(>':<EOL><INDENT>num_open_braces += <NUM_LIT:1><EOL><DEDENT>elif c == '<STR_LIT:)>':<EOL><INDENT>num_open_braces -= <NUM_LIT:1><EOL><DEDENT>if num_open_braces == <NUM_LIT:0>:<EOL><INDENT>return i<EOL><DEDENT><DEDENT>raise Exception("<STR_LIT>")<EOL>

Find the index of the closing braces for the opening braces
        at the start of the query string. Note that first character
        of input string must be an opening braces.


parts = []<EOL>num_open_braces = <NUM_LIT:0><EOL>delimiter = '<STR_LIT:U+002C>'<EOL>last_starting_index = <NUM_LIT:0><EOL>for i in range(len(query)):<EOL><INDENT>if query[i] == '<STR_LIT:(>':<EOL><INDENT>num_open_braces += <NUM_LIT:1><EOL><DEDENT>elif query[i] == '<STR_LIT:)>':<EOL><INDENT>num_open_braces -= <NUM_LIT:1><EOL><DEDENT>elif query[i] == delimiter and num_open_braces == <NUM_LIT:0>:<EOL><INDENT>parts.append(query[last_starting_index: i].strip())<EOL>last_starting_index = i + <NUM_LIT:1><EOL><DEDENT><DEDENT>parts.append(query[last_starting_index:].strip())<EOL>return parts<EOL>

The subparts are seperated by a comma. Make sure
        that commas inside the part themselves are not considered.