try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>metadata = topology_info["<STR_LIT>"]<EOL>self.write_success_response(metadata)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology = self.tracker.getTopologyByClusterRoleEnvironAndName(<EOL>cluster, role, environ, topology_name)<EOL>start_time = self.get_argument_starttime()<EOL>end_time = self.get_argument_endtime()<EOL>self.validateInterval(start_time, end_time)<EOL>query = self.get_argument_query()<EOL>metrics = yield tornado.gen.Task(self.executeMetricsQuery,<EOL>topology.tmaster, query, int(start_time), int(end_time))<EOL>self.write_success_response(metrics)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def executeMetricsQuery(self, tmaster, queryString, start_time, end_time, callback=None):<DEDENT>

query = Query(self.tracker)<EOL>metrics = yield query.execute_query(tmaster, queryString, start_time, end_time)<EOL>ret = {}<EOL>ret["<STR_LIT>"] = start_time<EOL>ret["<STR_LIT>"] = end_time<EOL>ret["<STR_LIT>"] = []<EOL>for metric in metrics:<EOL><INDENT>tl = {<EOL>"<STR_LIT:data>": metric.timeline<EOL>}<EOL>if metric.instance:<EOL><INDENT>tl["<STR_LIT>"] = metric.instance<EOL><DEDENT>ret["<STR_LIT>"].append(tl)<EOL><DEDENT>raise tornado.gen.Return(ret)<EOL>

Get the specified metrics for the given query in this topology.
Returns the following dict on success:
{
  "timeline": [{
    "instance": <instance>,
    "data": {
      <start_time> : <numeric value>,
      <start_time> : <numeric value>,
      ...
    }
  }, {
    ...
  }, ...
  "starttime": <numeric value>,
  "endtime": <numeric value>,
},

Returns the following dict on failure:
{
  "message": "..."
}


self.tracker = tracker<EOL>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>instance = self.get_argument_instance()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>result = yield getInstancePid(topology_info, instance)<EOL>self.write_success_response(result)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


try:<EOL><INDENT>cluster = self.get_argument_cluster()<EOL>role = self.get_argument_role()<EOL>environ = self.get_argument_environ()<EOL>topology_name = self.get_argument_topology()<EOL>topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)<EOL>physical_plan = topology_info["<STR_LIT>"]<EOL>self.write_success_response(physical_plan)<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>self.write_error_response(e)<EOL><DEDENT>


parser = argparse.ArgumentParser(<EOL>epilog='<STR_LIT>',<EOL>usage="<STR_LIT>",<EOL>add_help=False)<EOL>parser = add_titles(parser)<EOL>parser = add_arguments(parser)<EOL>ya_parser = argparse.ArgumentParser(<EOL>parents=[parser],<EOL>formatter_class=SubcommandHelpFormatter,<EOL>add_help=False)<EOL>subparsers = ya_parser.add_subparsers(<EOL>title="<STR_LIT>")<EOL>help_parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=False)<EOL>help_parser.set_defaults(help=True)<EOL>subparsers.add_parser(<EOL>'<STR_LIT:version>',<EOL>help='<STR_LIT>',<EOL>add_help=True)<EOL>return parser, ya_parser<EOL>


if not query:<EOL><INDENT>return None<EOL><DEDENT>if query[<NUM_LIT:0>] == '<STR_LIT:(>':<EOL><INDENT>index = self.find_closing_braces(query)<EOL>if index != len(query) - <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return self.parse_query_string(query[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL><DEDENT><DEDENT>start_index = query.find("<STR_LIT:(>")<EOL>if start_index < <NUM_LIT:0>:<EOL><INDENT>try:<EOL><INDENT>constant = float(query)<EOL>return constant<EOL><DEDENT>except ValueError:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>token = query[:start_index]<EOL>if token not in self.operators:<EOL><INDENT>raise Exception("<STR_LIT>" + token)<EOL><DEDENT>rest_of_the_query = query[start_index:]<EOL>braces_end_index = self.find_closing_braces(rest_of_the_query)<EOL>if braces_end_index != len(rest_of_the_query) - <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>parts = self.get_sub_parts(rest_of_the_query[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>if token == "<STR_LIT>":<EOL><INDENT>return self.operators[token](parts)<EOL><DEDENT>children = []<EOL>for part in parts:<EOL><INDENT>children.append(self.parse_query_string(part))<EOL><DEDENT>node = self.operators[token](children)<EOL>return node<EOL>

Returns a parse tree for the query, each of the node is a
        subclass of Operator. This is both a lexical as well as syntax analyzer step.


self.state_managers = statemanagerfactory.get_all_state_managers(self.config.statemgr_config)<EOL>try:<EOL><INDENT>for state_manager in self.state_managers:<EOL><INDENT>state_manager.start()<EOL><DEDENT><DEDENT>except Exception as ex:<EOL><INDENT>Log.error("<STR_LIT>" % ex)<EOL>traceback.print_exc()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>def on_topologies_watch(state_manager, topologies):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>")<EOL>Log.debug("<STR_LIT>" + str(topologies))<EOL>existingTopologies = self.getTopologiesForStateLocation(state_manager.name)<EOL>existingTopNames = map(lambda t: t.name, existingTopologies)<EOL>Log.debug("<STR_LIT>" + str(existingTopNames))<EOL>for name in existingTopNames:<EOL><INDENT>if name not in topologies:<EOL><INDENT>Log.info("<STR_LIT>",<EOL>name, state_manager.rootpath)<EOL>self.removeTopology(name, state_manager.name)<EOL><DEDENT><DEDENT>for name in topologies:<EOL><INDENT>if name not in existingTopNames:<EOL><INDENT>self.addNewTopology(state_manager, name)<EOL><DEDENT><DEDENT><DEDENT>for state_manager in self.state_managers:<EOL><INDENT>onTopologiesWatch = partial(on_topologies_watch, state_manager)<EOL>state_manager.get_topologies(onTopologiesWatch)<EOL><DEDENT>


packingPlan = {<EOL>"<STR_LIT:id>": "<STR_LIT>",<EOL>"<STR_LIT>": []<EOL>}<EOL>if not topology.packing_plan:<EOL><INDENT>return packingPlan<EOL><DEDENT>container_plans = topology.packing_plan.container_plans<EOL>containers = []<EOL>for container_plan in container_plans:<EOL><INDENT>instances = []<EOL>for instance_plan in container_plan.instance_plans:<EOL><INDENT>instance_resources = {"<STR_LIT>": instance_plan.resource.cpu,<EOL>"<STR_LIT>": instance_plan.resource.ram,<EOL>"<STR_LIT>": instance_plan.resource.disk}<EOL>instance = {"<STR_LIT>" : instance_plan.component_name,<EOL>"<STR_LIT>" : instance_plan.task_id,<EOL>"<STR_LIT>": instance_plan.component_index,<EOL>"<STR_LIT>": instance_resources}<EOL>instances.append(instance)<EOL><DEDENT>required_resource = {"<STR_LIT>": container_plan.requiredResource.cpu,<EOL>"<STR_LIT>": container_plan.requiredResource.ram,<EOL>"<STR_LIT>": container_plan.requiredResource.disk}<EOL>scheduled_resource = {}<EOL>if container_plan.scheduledResource:<EOL><INDENT>scheduled_resource = {"<STR_LIT>": container_plan.scheduledResource.cpu,<EOL>"<STR_LIT>": container_plan.scheduledResource.ram,<EOL>"<STR_LIT>": container_plan.scheduledResource.disk}<EOL><DEDENT>container = {"<STR_LIT:id>": container_plan.id,<EOL>"<STR_LIT>": instances,<EOL>"<STR_LIT>": required_resource,<EOL>"<STR_LIT>": scheduled_resource}<EOL>containers.append(container)<EOL><DEDENT>packingPlan["<STR_LIT:id>"] = topology.packing_plan.id<EOL>packingPlan["<STR_LIT>"] = containers<EOL>return json.dumps(packingPlan)<EOL>

Returns the representation of packing plan that will
be returned from Tracker.


execution_state = topology.execution_state<EOL>executionState = {<EOL>"<STR_LIT>": execution_state.cluster,<EOL>"<STR_LIT>": execution_state.environ,<EOL>"<STR_LIT>": execution_state.role,<EOL>"<STR_LIT>": topology.name,<EOL>"<STR_LIT>": execution_state.submission_time,<EOL>"<STR_LIT>": execution_state.submission_user,<EOL>"<STR_LIT>": execution_state.release_state.release_username,<EOL>"<STR_LIT>": execution_state.release_state.release_tag,<EOL>"<STR_LIT>": execution_state.release_state.release_version,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": [],<EOL>}<EOL>for extra_link in self.config.extra_links:<EOL><INDENT>link = extra_link.copy()<EOL>link["<STR_LIT:url>"] = self.config.get_formatted_url(executionState,<EOL>link[EXTRA_LINK_FORMATTER_KEY])<EOL>executionState["<STR_LIT>"].append(link)<EOL><DEDENT>return executionState<EOL>

Returns the repesentation of execution state that will
be returned from Tracker.


return filter(lambda t: t.state_manager_name == name, self.topologies)<EOL>

Returns all the topologies for a given state manager.


schedulerLocation = {<EOL>"<STR_LIT:name>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>}<EOL>if topology.scheduler_location:<EOL><INDENT>schedulerLocation["<STR_LIT:name>"] = topology.scheduler_location.topology_name<EOL>schedulerLocation["<STR_LIT>"] = topology.scheduler_location.http_endpoint<EOL>schedulerLocation["<STR_LIT>"] =topology.scheduler_location.job_page_link[<NUM_LIT:0>]if len(topology.scheduler_location.job_page_link) > <NUM_LIT:0> else "<STR_LIT>"<EOL><DEDENT>return schedulerLocation<EOL>

Returns the representation of scheduler location that will
be returned from Tracker.


bfs_queue = [[U, <NUM_LIT:0>]]  <EOL>visited = set()<EOL>max_depth = <NUM_LIT:0><EOL>while bfs_queue:<EOL><INDENT>[V, depth] = bfs_queue.pop()<EOL>if max_depth < depth:<EOL><INDENT>max_depth = depth<EOL><DEDENT>visited.add(V)<EOL>adj_set = self.edges[V]<EOL>for W in adj_set:<EOL><INDENT>if W not in visited:<EOL><INDENT>bfs_queue.append([W, depth + <NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT>return max_depth<EOL>

Returns the maximum distance between any vertex and U in the connected
component containing U
:param U:
:return:


return True<EOL>

Returns True. This is just usefule for checking that an object is an operator or not.


def __init__(self, componentName, metricName, instance, start, end, timeline):

self.componentName = componentName<EOL>self.metricName = metricName<EOL>self.instance = instance<EOL>self.start = start<EOL>self.end = end<EOL>self.timeline = self.floorTimestamps(start, end, timeline)<EOL>

Takes (componentName, metricname, instance, timeline)


def floorTimestamps(self, start, end, timeline):

ret = {}<EOL>for timestamp, value in timeline.items():<EOL><INDENT>ts = timestamp / <NUM_LIT> * <NUM_LIT><EOL>if start <= ts <= end:<EOL><INDENT>ret[ts] = value<EOL><DEDENT><DEDENT>return ret<EOL>


def setDefault(self, constant, start, end):

starttime = start / <NUM_LIT> * <NUM_LIT><EOL>if starttime < start:<EOL><INDENT>starttime += <NUM_LIT><EOL><DEDENT>endtime = end / <NUM_LIT> * <NUM_LIT><EOL>while starttime <= endtime:<EOL><INDENT>if starttime not in self.timeline or self.timeline[starttime] == <NUM_LIT:0>:<EOL><INDENT>self.timeline[starttime] = constant<EOL><DEDENT>starttime += <NUM_LIT><EOL><DEDENT>


bin_path = os.path.join(get_heron_tracker_dir(), BIN_DIR)<EOL>return bin_path<EOL>

This will provide heron tracker bin directory from .pex file.
:return: absolute path of heron lib directory


<EOL>pplan = topologyInfo["<STR_LIT>"]<EOL>stmgrId = pplan["<STR_LIT>"][instance_id]["<STR_LIT>"]<EOL>host = pplan["<STR_LIT>"][stmgrId]["<STR_LIT:host>"]<EOL>shell_port = pplan["<STR_LIT>"][stmgrId]["<STR_LIT>"]<EOL>return "<STR_LIT>" % (host, shell_port)<EOL>

Makes the http endpoint for the heron shell
if shell port is present, otherwise returns None.


printable = string.ascii_letters + string.digits + string.punctuation + '<STR_LIT:U+0020>'<EOL>return '<STR_LIT>'.join(ch if ch in printable else r'<STR_LIT>'.format(ord(ch)) for ch in bin_str)<EOL>


def make_shell_filestats_url(host, shell_port, path):

return "<STR_LIT>" % (host, shell_port, path)<EOL>

Make the url for filestats data in heron-shell
from the info stored in stmgr.


command = ['<STR_LIT>', '<STR_LIT>', x]<EOL>p = subprocess.Popen(command, stdout=subprocess.PIPE)<EOL>output, _ = p.communicate()<EOL>lines = output.split("<STR_LIT:\n>")<EOL>return lines[<NUM_LIT:0>]<EOL>

This will return the path of input arg for windows
:return: the path in windows


conf_path = os.path.join(get_heron_tracker_dir(), CONF_DIR)<EOL>return conf_path<EOL>

This will provide heron tracker conf directory from .pex file.
:return: absolute path of heron conf directory


def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length):

return "<STR_LIT>" %(host, shell_port, instance_id, offset, length)<EOL>

Make the url for log-file data in heron-shell
from the info stored in stmgr.


if sys.platform == '<STR_LIT>':<EOL><INDENT>return cygpath(x)<EOL><DEDENT>return identity(x)<EOL>

This will return the class path depending on the platform
:return: the class path


expanded_config_file_path = os.path.expanduser(config_file)<EOL>if not os.path.lexists(expanded_config_file_path):<EOL><INDENT>return None<EOL><DEDENT>configs = {}<EOL>with open(expanded_config_file_path, '<STR_LIT:r>') as f:<EOL><INDENT>configs = yaml.load(f)<EOL><DEDENT>return configs<EOL>

This will parse the config file for the tracker
:return: the config or None if the file is not found


return socket.gethostname()<EOL>


def run(command, parser, cl_args, unknown_args):

action = cl_args["<STR_LIT:action>"]<EOL>if action == Action.SET:<EOL><INDENT>call_editor(get_inventory_file(cl_args))<EOL>update_config_files(cl_args)<EOL><DEDENT>elif action == Action.CLUSTER:<EOL><INDENT>action_type = cl_args["<STR_LIT:type>"]<EOL>if action_type == Cluster.START:<EOL><INDENT>start_cluster(cl_args)<EOL><DEDENT>elif action_type == Cluster.STOP:<EOL><INDENT>if check_sure(cl_args, "<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"):<EOL><INDENT>stop_cluster(cl_args)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % action_type)<EOL><DEDENT><DEDENT>elif action == Action.TEMPLATE:<EOL><INDENT>update_config_files(cl_args)<EOL><DEDENT>elif action == Action.GET:<EOL><INDENT>action_type = cl_args["<STR_LIT:type>"]<EOL>if action_type == Get.SERVICE_URL:<EOL><INDENT>print(get_service_url(cl_args))<EOL><DEDENT>elif action_type == Get.HERON_UI_URL:<EOL><INDENT>print(get_heron_ui_url(cl_args))<EOL><DEDENT>elif action_type == Get.HERON_TRACKER_URL:<EOL><INDENT>print(get_heron_tracker_url(cl_args))<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % action_type)<EOL><DEDENT><DEDENT>elif action == Action.INFO:<EOL><INDENT>print_cluster_info(cl_args)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % action)<EOL><DEDENT>return SimpleResult(Status.Ok)<EOL>


scp = '<STR_LIT>' % (src, dest)<EOL>return scp<EOL>


cmd = "<STR_LIT>"<EOL>if not is_self(host):<EOL><INDENT>cmd = ssh_remote_execute(cmd, host, cl_args)<EOL><DEDENT>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (host, output))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>return output[<NUM_LIT:0>].strip("<STR_LIT:\n>")<EOL>


return socket.gethostbyname(socket.gethostname())<EOL>


Log.info("<STR_LIT>")<EOL>masters = roles[Role.MASTERS]<EOL>slaves = roles[Role.SLAVES]<EOL>tar_file = tempfile.NamedTemporaryFile(suffix="<STR_LIT>").name<EOL>Log.debug("<STR_LIT>" % (cl_args["<STR_LIT>"], tar_file))<EOL>make_tarfile(tar_file, cl_args["<STR_LIT>"])<EOL>dist_nodes = masters.union(slaves)<EOL>scp_package(tar_file, dist_nodes, cl_args)<EOL>


parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=True<EOL>)<EOL>cli_args.add_titles(parser)<EOL>parser_action = parser.add_subparsers()<EOL>parser_cluster = parser_action.add_parser(<EOL>Action.CLUSTER,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter,<EOL>)<EOL>parser_cluster.set_defaults(action=Action.CLUSTER)<EOL>parser_set = parser_action.add_parser(<EOL>Action.SET,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_set.set_defaults(action=Action.SET)<EOL>parser_template = parser_action.add_parser(<EOL>Action.TEMPLATE,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_template.set_defaults(action=Action.TEMPLATE)<EOL>parser_cluster.add_argument(<EOL>TYPE,<EOL>type=str,<EOL>choices={Cluster.START, Cluster.STOP},<EOL>help="<EOL>ces supports the following:<EOL>art     - Start standalone Heron cluster<EOL>op      - Stop standalone Heron cluster<EOL>)<EOL>parser_template.add_argument(<EOL>TYPE,<EOL>type=str,<EOL>choices={"<STR_LIT>"},<EOL>)<EOL>parser_get = parser_action.add_parser(<EOL>Action.GET,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_get.set_defaults(action=Action.GET)<EOL>parser_get.add_argument(<EOL>TYPE,<EOL>type=str,<EOL>choices={Get.SERVICE_URL, Get.HERON_TRACKER_URL, Get.HERON_UI_URL},<EOL>help="""<STR_LIT>"""<EOL>)<EOL>parser_info = parser_action.add_parser(<EOL>Action.INFO,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_info.set_defaults(action=Action.INFO)<EOL>add_additional_args([parser_set, parser_cluster, parser_template, parser_get, parser_info])<EOL>parser.set_defaults(subcommand='<STR_LIT>')<EOL>return parser<EOL>

Create a subparser for the standalone command
:param subparsers:
:return:


single_master = masters[<NUM_LIT:0>]<EOL>scheduler_config_actual = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>scheduler_config_template = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>template_file(scheduler_config_template, scheduler_config_actual,<EOL>{"<STR_LIT>": "<STR_LIT>" % single_master})<EOL>


roles = read_and_parse_roles(cl_args)<EOL>masters = roles[Role.MASTERS]<EOL>slaves = roles[Role.SLAVES]<EOL>zookeepers = roles[Role.ZOOKEEPERS]<EOL>Log.info("<STR_LIT>")<EOL>Log.info("<STR_LIT>" % list(masters))<EOL>Log.info("<STR_LIT>" % list(slaves))<EOL>Log.info("<STR_LIT>" % list(zookeepers))<EOL>if not masters:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>if not slaves:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>if not zookeepers:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>update_config_files(cl_args)<EOL>dist_nodes = list(masters.union(slaves))<EOL>if not (len(dist_nodes) == <NUM_LIT:1> and is_self(dist_nodes[<NUM_LIT:0>])):<EOL><INDENT>distribute_package(roles, cl_args)<EOL><DEDENT>start_master_nodes(masters, cl_args)<EOL>start_slave_nodes(slaves, cl_args)<EOL>start_api_server(masters, cl_args)<EOL>start_heron_tools(masters, cl_args)<EOL>Log.info("<STR_LIT>")<EOL>


roles = read_and_parse_roles(cl_args)<EOL>return "<STR_LIT>" % list(roles[Role.MASTERS])[<NUM_LIT:0>]<EOL>


single_master = list(masters)[<NUM_LIT:0>]<EOL>wait_for_master_to_start(single_master)<EOL>cmd = "<STR_LIT>"% (get_nomad_path(cl_args), get_heron_tools_job_file(cl_args))<EOL>Log.info("<STR_LIT>" % single_master)<EOL>if not is_self(single_master):<EOL><INDENT>cmd = ssh_remote_execute(cmd, single_master, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (single_master, output[<NUM_LIT:1>]))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>wait_for_job_to_start(single_master, "<STR_LIT>")<EOL>Log.info("<STR_LIT>")<EOL>


pids = []<EOL>for master in masters:<EOL><INDENT>Log.info("<STR_LIT>" % master)<EOL>cmd = "<STR_LIT>"% (get_nomad_path(cl_args), get_nomad_master_config_file(cl_args))<EOL>if not is_self(master):<EOL><INDENT>cmd = ssh_remote_execute(cmd, master, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>pids.append({"<STR_LIT>": pid, "<STR_LIT>": master})<EOL><DEDENT>errors = []<EOL>for entry in pids:<EOL><INDENT>pid = entry["<STR_LIT>"]<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>errors.append("<STR_LIT>" % (entry["<STR_LIT>"], output[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>if errors:<EOL><INDENT>for error in errors:<EOL><INDENT>Log.error(error)<EOL><DEDENT>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>


if is_self(ip_addr):<EOL><INDENT>return get_self_hostname()<EOL><DEDENT>cmd = "<STR_LIT>"<EOL>ssh_cmd = ssh_remote_execute(cmd, ip_addr, cl_args)<EOL>pid = subprocess.Popen(ssh_cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (ip_addr, output))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>return output[<NUM_LIT:0>].strip("<STR_LIT:\n>")<EOL>


with tarfile.open(output_filename, "<STR_LIT>") as tar:<EOL><INDENT>tar.add(source_dir, arcname=os.path.basename(source_dir))<EOL><DEDENT>


def scp_package(package_file, destinations, cl_args):

pids = []<EOL>for dest in destinations:<EOL><INDENT>if is_self(dest):<EOL><INDENT>continue<EOL><DEDENT>Log.info("<STR_LIT>" % dest)<EOL>file_path = "<STR_LIT>"<EOL>dest_file_path = "<STR_LIT>" % (dest, file_path)<EOL>remote_cmd = "<STR_LIT>""<STR_LIT>" % (file_path)<EOL>cmd = '<STR_LIT>'% (scp_cmd(package_file, dest_file_path, cl_args),<EOL>ssh_remote_execute(remote_cmd, dest, cl_args))<EOL>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>pids.append({"<STR_LIT>": pid, "<STR_LIT>": dest})<EOL><DEDENT>errors = []<EOL>for entry in pids:<EOL><INDENT>pid = entry["<STR_LIT>"]<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>errors.append("<STR_LIT>" % (entry["<STR_LIT>"], output[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>if errors:<EOL><INDENT>for error in errors:<EOL><INDENT>Log.error(error)<EOL><DEDENT>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>


Log.info("<STR_LIT>")<EOL>roles = read_and_parse_roles(cl_args)<EOL>masters = roles[Role.MASTERS]<EOL>slaves = roles[Role.SLAVES]<EOL>dist_nodes = masters.union(slaves)<EOL>if masters:<EOL><INDENT>try:<EOL><INDENT>single_master = list(masters)[<NUM_LIT:0>]<EOL>jobs = get_jobs(cl_args, single_master)<EOL>for job in jobs:<EOL><INDENT>job_id = job["<STR_LIT>"]<EOL>Log.info("<STR_LIT>" % job_id)<EOL>delete_job(cl_args, job_id, single_master)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL>Log.debug(sys.exc_info()[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>for node in dist_nodes:<EOL><INDENT>Log.info("<STR_LIT>" % node)<EOL>if not is_self(node):<EOL><INDENT>cmd = "<STR_LIT>""<STR_LIT>"<EOL>cmd = ssh_remote_execute(cmd, node, cl_args)<EOL><DEDENT>else:<EOL><INDENT>cmd = "<STR_LIT>""<STR_LIT>"<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>Log.info("<STR_LIT>" % node)<EOL>cmd = "<STR_LIT>"<EOL>if not is_self(node):<EOL><INDENT>cmd = ssh_remote_execute(cmd, node, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL><DEDENT>


def run(handlers, command, parser, command_args, unknown_args):

if command in handlers:<EOL><INDENT>return handlers[command].run(command, parser, command_args, unknown_args)<EOL><DEDENT>else:<EOL><INDENT>err_context = '<STR_LIT>' % command<EOL>return result.SimpleResult(result.Status.InvocationError, err_context)<EOL><DEDENT>

Run the command
:param command:
:param parser:
:param command_args:
:param unknown_args:
:return:


return {<EOL>'<STR_LIT>': standalone,<EOL>}<EOL>

Create a map of command names and handlers


parser = argparse.ArgumentParser(<EOL>prog='<STR_LIT>',<EOL>epilog=HELP_EPILOG,<EOL>formatter_class=config.SubcommandHelpFormatter,<EOL>add_help=True)<EOL>subparsers = parser.add_subparsers(<EOL>title="<STR_LIT>",<EOL>metavar='<STR_LIT>')<EOL>command_list = sorted(command_handlers.items())<EOL>for command in command_list:<EOL><INDENT>command[<NUM_LIT:1>].create_parser(subparsers)<EOL><DEDENT>return parser<EOL>


<EOL>check_environment()<EOL>parser = create_parser(handlers)<EOL>if len(sys.argv[<NUM_LIT:1>:]) == <NUM_LIT:0>:<EOL><INDENT>parser.print_help()<EOL>return <NUM_LIT:0><EOL><DEDENT>sys.argv = config.insert_bool_values(sys.argv)<EOL>try:<EOL><INDENT>args, unknown_args = parser.parse_known_args()<EOL><DEDENT>except ValueError as ex:<EOL><INDENT>Log.error("<STR_LIT>", str(ex))<EOL>Log.debug(traceback.format_exc())<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>command_line_args = vars(args)<EOL>log.set_logging_level(command_line_args)<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>command = command_line_args['<STR_LIT>']<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>results = run(handlers, command, parser, command_line_args, unknown_args)<EOL>return <NUM_LIT:0> if result.is_successful(results) else <NUM_LIT:1><EOL>


for cur_file in files:<EOL><INDENT>if os.path.isdir(cur_file):<EOL><INDENT>shutil.rmtree(cur_file)<EOL><DEDENT>else:<EOL><INDENT>shutil.rmtree(os.path.dirname(cur_file))<EOL><DEDENT><DEDENT>


configcommand = cl_args.get('<STR_LIT>', None)<EOL>if configcommand == '<STR_LIT>':<EOL><INDENT>return _set(cl_args)<EOL><DEDENT>elif configcommand == '<STR_LIT>':<EOL><INDENT>return _unset(cl_args)<EOL><DEDENT>else:<EOL><INDENT>return _list(cl_args)<EOL><DEDENT>

:param command:
:param parser:
:param args:
:param unknown_args:
:return:


config_path = config.get_heron_cluster_conf_dir(cluster, config_path)<EOL>if not os.path.isdir(config_path):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Check the cluster definition for direct mode
:param cluster:
:param config_path:
:return:


client_confs = dict()<EOL>client_confs[cluster] = cliconfig.cluster_config(cluster)<EOL>if cl_args.get('<STR_LIT>', None):<EOL><INDENT>client_confs[cluster]['<STR_LIT>'] = cl_args['<STR_LIT>']<EOL><DEDENT>return client_confs<EOL>

Read the cluster definition for server mode
:param cluster:
:param cl_args:
:param config_file:
:return:


Create a subparser for the submit command
:param subparsers:
:return:


def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name):

service_apiurl = cl_args['<STR_LIT>'] + rest.ROUTE_SIGNATURES['<STR_LIT>'][<NUM_LIT:1>]<EOL>service_method = rest.ROUTE_SIGNATURES['<STR_LIT>'][<NUM_LIT:0>]<EOL>data = dict(<EOL>name=topology_name,<EOL>cluster=cl_args['<STR_LIT>'],<EOL>role=cl_args['<STR_LIT>'],<EOL>environment=cl_args['<STR_LIT>'],<EOL>user=cl_args['<STR_LIT>'],<EOL>)<EOL>Log.info("<STR_LIT>" + str(cl_args))<EOL>overrides = dict()<EOL>if '<STR_LIT>' in cl_args:<EOL><INDENT>overrides = config.parse_override_config(cl_args['<STR_LIT>'])<EOL><DEDENT>if overrides:<EOL><INDENT>data.update(overrides)<EOL><DEDENT>if cl_args['<STR_LIT>']:<EOL><INDENT>data["<STR_LIT>"] = True<EOL><DEDENT>files = dict(<EOL>definition=open(topology_defn_file, '<STR_LIT:rb>'),<EOL>topology=open(topology_file, '<STR_LIT:rb>'),<EOL>)<EOL>err_ctxt = "<STR_LIT>" % (topology_name, launch_mode_msg(cl_args))<EOL>succ_ctxt = "<STR_LIT>" % (topology_name, launch_mode_msg(cl_args))<EOL>try:<EOL><INDENT>r = service_method(service_apiurl, data=data, files=files)<EOL>ok = r.status_code is requests.codes.ok<EOL>created = r.status_code is requests.codes.created<EOL>s = Status.Ok if created or ok else Status.HeronError<EOL>if s is Status.HeronError:<EOL><INDENT>Log.error(r.json().get('<STR_LIT:message>', "<STR_LIT>" % r.status_code))<EOL><DEDENT>elif ok:<EOL><INDENT>print(r.json().get("<STR_LIT>"))<EOL><DEDENT><DEDENT>except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:<EOL><INDENT>Log.error(err)<EOL>return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)<EOL><DEDENT>return SimpleResult(s, err_ctxt, succ_ctxt)<EOL>

Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param topology_file:
:param topology_defn_file:
:param topology_name:
:return:


def launch_topologies(cl_args, topology_file, tmp_dir):

<EOL>defn_files = glob.glob(tmp_dir + '<STR_LIT>')<EOL>if len(defn_files) == <NUM_LIT:0>:<EOL><INDENT>return SimpleResult(Status.HeronError, "<STR_LIT>" % tmp_dir)<EOL><DEDENT>results = []<EOL>for defn_file in defn_files:<EOL><INDENT>topology_defn = topology_pb2.Topology()<EOL>try:<EOL><INDENT>handle = open(defn_file, "<STR_LIT:rb>")<EOL>topology_defn.ParseFromString(handle.read())<EOL>handle.close()<EOL><DEDENT>except Exception as e:<EOL><INDENT>err_context = "<STR_LIT>" % (defn_file, e)<EOL>return SimpleResult(Status.HeronError, err_context)<EOL><DEDENT>Log.info("<STR_LIT>", topology_defn.name, launch_mode_msg(cl_args))<EOL>if cl_args['<STR_LIT>'] == config.SERVER_MODE:<EOL><INDENT>res = launch_topology_server(<EOL>cl_args, topology_file, defn_file, topology_defn.name)<EOL><DEDENT>else:<EOL><INDENT>res = launch_a_topology(<EOL>cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)<EOL><DEDENT>results.append(res)<EOL><DEDENT>return results<EOL>

Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)


if cl_args['<STR_LIT>']:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>return "<STR_LIT>"<EOL>

Depending on the mode of launching a topology provide a message
:param cl_args:
:return:


<EOL>component_parallelism = cl_args['<STR_LIT>']<EOL>runtime_configs = cl_args['<STR_LIT>']<EOL>container_number = cl_args['<STR_LIT>']<EOL>if (component_parallelism and runtime_configs) or (container_number and runtime_configs):<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>")<EOL><DEDENT>dict_extra_args = {}<EOL>nothing_set = True<EOL>if component_parallelism:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': component_parallelism})<EOL>nothing_set = False<EOL><DEDENT>if container_number:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': container_number})<EOL>nothing_set = False<EOL><DEDENT>if runtime_configs:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': runtime_configs})<EOL>nothing_set = False<EOL><DEDENT>if nothing_set:<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>")<EOL><DEDENT>if cl_args['<STR_LIT>']:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': True})<EOL>if '<STR_LIT>' in cl_args:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': cl_args["<STR_LIT>"]})<EOL><DEDENT><DEDENT>return dict_extra_args<EOL>


opt_list = []<EOL>for (key, value) in config_opts.items():<EOL><INDENT>opt_list.append('<STR_LIT>' % (key, value))<EOL><DEDENT>all_opts = ('<STR_LIT:U+002C>'.join(opt_list)).replace('<STR_LIT:U+0020>', '<STR_LIT>')<EOL>return all_opts<EOL>

Get config opts from the global variable
:return:


global config_opts<EOL>if k in config_opts:<EOL><INDENT>return config_opts[k]<EOL><DEDENT>return None<EOL>

Get config opts from the config map
:param k:
:return:


def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):

topology_name = cl_args['<STR_LIT>']<EOL>new_args = [<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", config.get_heron_dir(),<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", config.get_heron_release_file(),<EOL>"<STR_LIT>", topology_name,<EOL>"<STR_LIT>", command,<EOL>]<EOL>new_args += extra_args<EOL>lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())<EOL>lib_jars += extra_lib_jars<EOL>if Log.getEffectiveLevel() == logging.DEBUG:<EOL><INDENT>new_args.append("<STR_LIT>")<EOL><DEDENT>result = execute.heron_class(<EOL>'<STR_LIT>',<EOL>lib_jars,<EOL>extra_jars=[],<EOL>args=new_args<EOL>)<EOL>err_msg = "<STR_LIT>" % (action, topology_name)<EOL>succ_msg = "<STR_LIT>" % (action, topology_name)<EOL>result.add_context(err_msg, succ_msg)<EOL>return result<EOL>

helper function to take action on topologies
:param command:
:param cl_args:
:param action:        description of action taken
:return:


def run(command, parser, args, unknown_args):

<EOL>command_help = args['<STR_LIT>']<EOL>if command_help == '<STR_LIT>':<EOL><INDENT>parser.print_help()<EOL>return SimpleResult(Status.Ok)<EOL><DEDENT>subparser = config.get_subparser(parser, command_help)<EOL>if subparser:<EOL><INDENT>print(subparser.format_help())<EOL>return SimpleResult(Status.Ok)<EOL><DEDENT>else:<EOL><INDENT>Log.error("<STR_LIT>", command_help)<EOL>return SimpleResult(Status.InvocationError)<EOL><DEDENT>


<EOL>check_environment()<EOL>parser = create_parser(handlers)<EOL>if len(sys.argv[<NUM_LIT:1>:]) == <NUM_LIT:0>:<EOL><INDENT>parser.print_help()<EOL>return <NUM_LIT:0><EOL><DEDENT>sys.argv = config.insert_bool_values(sys.argv)<EOL>try:<EOL><INDENT>args, unknown_args = parser.parse_known_args()<EOL><DEDENT>except ValueError as ex:<EOL><INDENT>Log.error("<STR_LIT>", str(ex))<EOL>Log.debug(traceback.format_exc())<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>command_line_args = vars(args)<EOL>log.set_logging_level(command_line_args)<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>mmand to be execute<EOL>command = command_line_args['<STR_LIT>']<EOL>is_local_command = command in local_commands<EOL>if command == '<STR_LIT:version>':<EOL><INDENT>results = run(handlers, command, parser, command_line_args, unknown_args)<EOL>return <NUM_LIT:0> if result.is_successful(results) else <NUM_LIT:1><EOL><DEDENT>if not is_local_command:<EOL><INDENT>log.set_logging_level(command_line_args)<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>command_line_args = extract_common_args(command, parser, command_line_args)<EOL>command_line_args = deployment_mode(command, parser, command_line_args)<EOL>if not command_line_args:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if command_line_args['<STR_LIT>'] == config.DIRECT_MODE and command != "<STR_LIT:version>":<EOL><INDENT>cleaned_up_files.append(command_line_args['<STR_LIT>'])<EOL>atexit.register(cleanup, cleaned_up_files)<EOL><DEDENT><DEDENT>Log.debug("<STR_LIT>", command_line_args)<EOL>start = time.time()<EOL>results = run(handlers, command, parser, command_line_args, unknown_args)<EOL>if not is_local_command:<EOL><INDENT>result.render(results)<EOL><DEDENT>end = time.time()<EOL>if not is_local_command:<EOL><INDENT>sys.stdout.flush()<EOL>Log.debug('<STR_LIT>', (end - start))<EOL><DEDENT>return <NUM_LIT:0> if result.is_successful(results) else <NUM_LIT:1><EOL>


Log.debug("<STR_LIT>", cl_args)<EOL>return cli_helper.run(command, cl_args, "<STR_LIT>")<EOL>

:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:


return cli_helper.create_parser(subparsers, '<STR_LIT>', '<STR_LIT>')<EOL>


<EOL>assert self.status is not None<EOL>if self.status == Status.Ok:<EOL><INDENT>self._do_log(Log.info, stdout)<EOL><DEDENT>elif self.status == Status.HeronError:<EOL><INDENT>self._do_log(Log.error, stdout)<EOL><DEDENT>elif self.status == Status.DryRun:<EOL><INDENT>self._do_print(sys.stdout, stdout)<EOL><DEDENT>elif self.status == Status.InvocationError:<EOL><INDENT>self._do_print(sys.stdout, stdout)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>" %(self.status.value, list(Status)))<EOL><DEDENT>

render stdout of shelled-out process
            stdout always contains information Java process wants to
            propagate back to cli, so we do special rendering here
        :param stdout: all lines from shelled-out process
        :return:


retcode = self.process.poll()<EOL>if retcode is not None and status_type(retcode) == Status.InvocationError:<EOL><INDENT>self._do_log(Log.error, stderr_line)<EOL><DEDENT>else:<EOL><INDENT>self._do_print(sys.stderr, stderr_line)<EOL><DEDENT>

render stderr of shelled-out process
            stderr could be error message of failure of invoking process or
            normal stderr output from successfully shelled-out process.
            In the first case, ``Popen'' should fail fast and we should be able to
            get return code immediately. We then render the failure message.
            In the second case, we simply print stderr line in stderr.
            The way to handle the first case is shaky but should be the best we can
            do since we have conflicts of design goals here.
        :param stderr_line: one line from shelled-out process
        :return:


def add_context(self, err_context, succ_context=None):

self.err_context = err_context<EOL>self.succ_context = succ_context<EOL>

Prepend msg to add some context information

        :param pmsg: context info
        :return: None


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>default=False,<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


jars = [<EOL>os.path.join(config.get_heron_lib_dir(), "<STR_LIT>", "<STR_LIT:*>")<EOL>]<EOL>return jars<EOL>


Log.debug("<STR_LIT>", cl_args)<EOL>container_id = cl_args['<STR_LIT>']<EOL>if cl_args['<STR_LIT>'] == config.SERVER_MODE:<EOL><INDENT>dict_extra_args = {"<STR_LIT>": str(container_id)}<EOL>return cli_helper.run_server(command, cl_args, "<STR_LIT>", extra_args=dict_extra_args)<EOL><DEDENT>else:<EOL><INDENT>list_extra_args = ["<STR_LIT>", str(container_id)]<EOL>return cli_helper.run_direct(command, cl_args, "<STR_LIT>", extra_args=list_extra_args)<EOL><DEDENT>


def run(command, parser, known_args, unknown_args):

config.print_build_info()<EOL>return True<EOL>


try:<EOL><INDENT>result = tracker_access.get_cluster_topologies(cluster)<EOL>if not result:<EOL><INDENT>Log.error('<STR_LIT>' % cluster)<EOL>return False<EOL><DEDENT>result = result[cluster]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>table, header, rest_count = to_table(result)<EOL>print('<STR_LIT>' % cluster)<EOL>if rest_count:<EOL><INDENT>print('<STR_LIT>' % rest_count)<EOL><DEDENT>print(tabulate(table, headers=header))<EOL>return True<EOL>


def show_cluster_role_env(cl_args, cluster, role, env):

try:<EOL><INDENT>result = tracker_access.get_cluster_role_env_topologies(cluster, role, env)<EOL>if not result:<EOL><INDENT>Log.error('<STR_LIT>' % '<STR_LIT:/>'.join([cluster, role, env]))<EOL>return False<EOL><DEDENT>result = result[cluster]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>table, header, rest_count = to_table(result)<EOL>print('<STR_LIT>' % (cluster, role, env))<EOL>if rest_count:<EOL><INDENT>print('<STR_LIT>' % rest_count)<EOL><DEDENT>print(tabulate(table, headers=header))<EOL>return True<EOL>


max_count = <NUM_LIT:20><EOL>table, count = [], <NUM_LIT:0><EOL>for role, envs_topos in result.items():<EOL><INDENT>for env, topos in envs_topos.items():<EOL><INDENT>for topo in topos:<EOL><INDENT>count += <NUM_LIT:1><EOL>if count > max_count:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>table.append([role, env, topo])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>header = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>rest_count = <NUM_LIT:0> if count <= max_count else count - max_count<EOL>return table, header, rest_count<EOL>


def show_cluster_role(cl_args, cluster, role):

try:<EOL><INDENT>result = tracker_access.get_cluster_role_topologies(cluster, role)<EOL>if not result:<EOL><INDENT>Log.error('<STR_LIT>' % '<STR_LIT:/>'.join([cluster, role]))<EOL>return False<EOL><DEDENT>result = result[cluster]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>table, header, rest_count = to_table(result)<EOL>print('<STR_LIT>' % (cluster, role))<EOL>if rest_count:<EOL><INDENT>print('<STR_LIT>' % rest_count)<EOL><DEDENT>print(tabulate(table, headers=header))<EOL>return True<EOL>


location = cl_args['<STR_LIT>'].split('<STR_LIT:/>')<EOL>if len(location) == <NUM_LIT:1>:<EOL><INDENT>return show_cluster(cl_args, *location)<EOL><DEDENT>elif len(location) == <NUM_LIT:2>:<EOL><INDENT>return show_cluster_role(cl_args, *location)<EOL><DEDENT>elif len(location) == <NUM_LIT:3>:<EOL><INDENT>return show_cluster_role_env(cl_args, *location)<EOL><DEDENT>else:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>


try:<EOL><INDENT>topo_loc = cl_args['<STR_LIT>'].split('<STR_LIT:/>')<EOL>topo_loc.append(cl_args['<STR_LIT>'])<EOL>if len(topo_loc) != <NUM_LIT:4>:<EOL><INDENT>raise<EOL><DEDENT>return topo_loc<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>raise<EOL><DEDENT>


def run_components(command, parser, cl_args, unknown_args):

return run(cl_args, '<STR_LIT:all>')<EOL>


inputs, outputs = defaultdict(list), defaultdict(list)<EOL>for ctype, component in components.items():<EOL><INDENT>if ctype == '<STR_LIT>':<EOL><INDENT>for component_name, component_info in component.items():<EOL><INDENT>for input_stream in component_info['<STR_LIT>']:<EOL><INDENT>input_name = input_stream['<STR_LIT>']<EOL>inputs[component_name].append(input_name)<EOL>outputs[input_name].append(component_name)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>info = []<EOL>spouts_instance = topo_info['<STR_LIT>']['<STR_LIT>']<EOL>bolts_instance = topo_info['<STR_LIT>']['<STR_LIT>']<EOL>for ctype, component in components.items():<EOL><INDENT>if ctype == "<STR_LIT>":<EOL><INDENT>continue<EOL><DEDENT>for component_name, component_info in component.items():<EOL><INDENT>row = [ctype[:-<NUM_LIT:1>], component_name]<EOL>if ctype == '<STR_LIT>':<EOL><INDENT>row.append(len(spouts_instance[component_name]))<EOL><DEDENT>else:<EOL><INDENT>row.append(len(bolts_instance[component_name]))<EOL><DEDENT>row.append('<STR_LIT:U+002C>'.join(inputs.get(component_name, ['<STR_LIT:->'])))<EOL>row.append('<STR_LIT:U+002C>'.join(outputs.get(component_name, ['<STR_LIT:->'])))<EOL>info.append(row)<EOL><DEDENT><DEDENT>header = ['<STR_LIT:type>', '<STR_LIT:name>', '<STR_LIT>', '<STR_LIT:input>', '<STR_LIT>']<EOL>return info, header<EOL>

normalize raw logical plan info to table


def run_containers(command, parser, cl_args, unknown_args):

cluster, role, env = cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>']<EOL>topology = cl_args['<STR_LIT>']<EOL>container_id = cl_args['<STR_LIT:id>']<EOL>try:<EOL><INDENT>result = tracker_access.get_topology_info(cluster, env, topology, role)<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>containers = result['<STR_LIT>']['<STR_LIT>']<EOL>all_bolts, all_spouts = set(), set()<EOL>for _, bolts in result['<STR_LIT>']['<STR_LIT>'].items():<EOL><INDENT>all_bolts = all_bolts | set(bolts)<EOL><DEDENT>for _, spouts in result['<STR_LIT>']['<STR_LIT>'].items():<EOL><INDENT>all_spouts = all_spouts | set(spouts)<EOL><DEDENT>stmgrs = containers.keys()<EOL>stmgrs.sort()<EOL>if container_id is not None:<EOL><INDENT>try:<EOL><INDENT>normalized_cid = container_id - <NUM_LIT:1><EOL>if normalized_cid < <NUM_LIT:0>:<EOL><INDENT>raise<EOL><DEDENT>stmgrs = [stmgrs[normalized_cid]]<EOL><DEDENT>except:<EOL><INDENT>Log.error('<STR_LIT>' % container_id)<EOL>return False<EOL><DEDENT><DEDENT>table = []<EOL>for sid, name in enumerate(stmgrs):<EOL><INDENT>cid = sid + <NUM_LIT:1><EOL>host = containers[name]["<STR_LIT:host>"]<EOL>port = containers[name]["<STR_LIT:port>"]<EOL>pid = containers[name]["<STR_LIT>"]<EOL>instances = containers[name]["<STR_LIT>"]<EOL>bolt_nums = len([instance for instance in instances if instance in all_bolts])<EOL>spout_nums = len([instance for instance in instances if instance in all_spouts])<EOL>table.append([cid, host, port, pid, bolt_nums, spout_nums, len(instances)])<EOL><DEDENT>headers = ["<STR_LIT>", "<STR_LIT:host>", "<STR_LIT:port>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>sys.stdout.flush()<EOL>print(tabulate(table, headers=headers))<EOL>return True<EOL>


try:<EOL><INDENT>topo_loc = cl_args['<STR_LIT>'].split('<STR_LIT:/>')<EOL>topo_name = cl_args['<STR_LIT>']<EOL>topo_loc.append(topo_name)<EOL>if len(topo_loc) != <NUM_LIT:4>:<EOL><INDENT>raise<EOL><DEDENT>return topo_loc<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>raise<EOL><DEDENT>


<EOL>if command == '<STR_LIT>':<EOL><INDENT>return clusters.run(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return topologies.run(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return physicalplan.run_containers(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return physicalplan.run_metrics(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return logicalplan.run_components(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return logicalplan.run_spouts(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return logicalplan.run_bolts(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return help.run(command, *args)<EOL><DEDENT>elif command == '<STR_LIT:version>':<EOL><INDENT>return version.run(command, *args)<EOL><DEDENT>return <NUM_LIT:1><EOL>


def extract_common_args(command, parser, cl_args):

try:<EOL><INDENT>cluster_role_env = cl_args['<STR_LIT>']<EOL>config_path = cl_args['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>subparser = config.get_subparser(parser, command)<EOL>print(subparser.format_help())<EOL>return dict()<EOL><DEDENT>cluster = config.get_heron_cluster(cluster_role_env)<EOL>config_path = config.get_heron_cluster_conf_dir(cluster, config_path)<EOL>new_cl_args = dict()<EOL>try:<EOL><INDENT>cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:0>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:1>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:2>]<EOL>new_cl_args['<STR_LIT>'] = config_path<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>", str(e))<EOL>return dict()<EOL><DEDENT>cl_args.update(new_cl_args)<EOL>return cl_args<EOL>


try:<EOL><INDENT>clusters = tracker_access.get_clusters()<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>print('<STR_LIT>')<EOL>for cluster in clusters:<EOL><INDENT>print('<STR_LIT>' % cluster)<EOL><DEDENT>return True<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>)<EOL>return parser<EOL>

add argument that specifies topology name


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>type=str)<EOL>return parser<EOL>

add optional argument that specifies component name


parser.add_argument(<EOL>'<STR_LIT>', help='<STR_LIT>', action='<STR_LIT:store_true>')<EOL>return parser<EOL>

add optional argument that displays spout only


parser.add_argument(<EOL>'<STR_LIT>', help='<STR_LIT>', type=str,<EOL>metavar='<STR_LIT>')<EOL>return parser<EOL>

add argument that specifies topologies location


parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + DEFAULT_TRACKER_URL + '<STR_LIT>',<EOL>type=str, default=DEFAULT_TRACKER_URL)<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>type=bool,<EOL>default=False)<EOL>return parser<EOL>


args1 = insert_bool('<STR_LIT>', command_line_args)<EOL>return args1<EOL>


<EOL>self.connection_closed = True<EOL>


def get(self, cluster, environ, topology, comp_name, instance):

:param cluster:
:param environ:
:param topology:
:param comp_name:
:param instance:
:return:


def get(self, cluster, environ, topology):

:param cluster:
:param environ:
:param topology:
:return:


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology):<DEDENT>

<EOL>execution_state = yield access.get_execution_state(cluster, environ, topology)<EOL>scheduler_location = yield access.get_scheduler_location(cluster, environ, topology)<EOL>job_page_link = scheduler_location["<STR_LIT>"]<EOL>launched_at = datetime.utcfromtimestamp(execution_state['<STR_LIT>'])<EOL>launched_time = launched_at.strftime('<STR_LIT>')<EOL>options = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>execution_state=execution_state,<EOL>launched=launched_time,<EOL>status="<STR_LIT>" if random.randint(<NUM_LIT:0>, <NUM_LIT:1>) else "<STR_LIT>",<EOL>active="<STR_LIT>",<EOL>job_page_link=job_page_link,<EOL>function=common.className,<EOL>baseUrl=self.baseUrl<EOL>)<EOL>self.render("<STR_LIT>", **options)<EOL>


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology, container):<DEDENT>

offset = self.get_argument("<STR_LIT>")<EOL>length = self.get_argument("<STR_LIT>")<EOL>path = self.get_argument("<STR_LIT:path>")<EOL>data = yield access.get_container_file_data(cluster, environ, topology, container, path,<EOL>offset, length)<EOL>self.write(data)<EOL>self.finish()<EOL>

:param cluster:
:param environ:
:param topology:
:param container:
:return:


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology, instance):<DEDENT>

pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.get_instance_pid(<EOL>cluster, environ, topology, instance)))<EOL>self.write('<STR_LIT>' % (<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>

:param cluster:
:param environ:
:param topology:
:param instance:
:return:


<EOL>topologies = yield access.get_topologies_states()<EOL>result = dict()<EOL>for cluster, cluster_value in topologies.items():<EOL><INDENT>result[cluster] = dict()<EOL>for environ, environ_value in cluster_value.items():<EOL><INDENT>result[cluster][environ] = dict()<EOL>for topology, topology_value in environ_value.items():<EOL><INDENT>if "<STR_LIT>" not in topology_value or topology_value["<STR_LIT>"] is None:<EOL><INDENT>continue<EOL><DEDENT>if "<STR_LIT>" in topology_value:<EOL><INDENT>topology_value["<STR_LIT>"] = topology_value["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>topology_value["<STR_LIT>"] = '<STR_LIT:->'<EOL><DEDENT>result[cluster][environ][topology] = topology_value<EOL><DEDENT><DEDENT><DEDENT>self.write(result)<EOL>


pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.run_instance_jmap(<EOL>cluster, environ, topology, instance)))<EOL>notes = "<STR_LIT>".join([<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % host<EOL>])<EOL>self.write('<STR_LIT>' % (<EOL>notes,<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>


start_time = time.time()<EOL>estate = yield access.get_execution_state(cluster, environ, topology)<EOL>result_map = dict(<EOL>status="<STR_LIT:success>",<EOL>message="<STR_LIT>",<EOL>version=common.VERSION,<EOL>executiontime=time.time() - start_time,<EOL>result=estate<EOL>)<EOL>self.write(result_map)<EOL>


pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.get_instance_jstack(<EOL>cluster, environ, topology, instance)))<EOL>self.write('<STR_LIT>' % (<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>


start_time = time.time()<EOL>lplan = yield access.get_logical_plan(cluster, environ, topology)<EOL>result = dict(<EOL>status="<STR_LIT:success>",<EOL>message="<STR_LIT>",<EOL>version=common.VERSION,<EOL>executiontime=time.time() - start_time,<EOL>result=lplan<EOL>)<EOL>self.write(result)<EOL>


cluster = self.get_argument("<STR_LIT>")<EOL>environ = self.get_argument("<STR_LIT>")<EOL>topology = self.get_argument("<STR_LIT>")<EOL>component = self.get_argument("<STR_LIT>", default=None)<EOL>metricnames = self.get_arguments("<STR_LIT>")<EOL>instances = self.get_arguments("<STR_LIT>")<EOL>interval = self.get_argument("<STR_LIT>", default=-<NUM_LIT:1>)<EOL>time_range = (<NUM_LIT:0>, interval)<EOL>compnames = [component] if component else (yield access.get_comps(cluster, environ, topology))<EOL>futures = {}<EOL>for comp in compnames:<EOL><INDENT>future = access.get_comp_metrics(<EOL>cluster, environ, topology, comp, instances,<EOL>metricnames, time_range)<EOL>futures[comp] = future<EOL><DEDENT>results = yield futures<EOL>self.write(results[component] if component else results)<EOL>


<EOL>now = int(time.time())<EOL>time_slots = dict()<EOL>for key, value in ranges.items():<EOL><INDENT>time_slots[key] = (now - value[<NUM_LIT:0>], now - value[<NUM_LIT:1>], value[<NUM_LIT:2>])<EOL><DEDENT>return (now, time_slots)<EOL>


parser = argparse.ArgumentParser(<EOL>epilog='<STR_LIT>',<EOL>usage="<STR_LIT>",<EOL>add_help=False)<EOL>parser = add_titles(parser)<EOL>parser = add_arguments(parser)<EOL>child_parser = argparse.ArgumentParser(<EOL>parents=[parser],<EOL>formatter_class=SubcommandHelpFormatter,<EOL>add_help=False)<EOL>subparsers = child_parser.add_subparsers(<EOL>title="<STR_LIT>")<EOL>help_parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=False)<EOL>version_parser = subparsers.add_parser(<EOL>'<STR_LIT:version>',<EOL>help='<STR_LIT>',<EOL>add_help=True)<EOL>help_parser.set_defaults(help=True)<EOL>version_parser.set_defaults(version=True)<EOL>return (parser, child_parser)<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + consts.DEFAULT_TRACKER_URL + '<STR_LIT>',<EOL>default=consts.DEFAULT_TRACKER_URL)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + consts.DEFAULT_ADDRESS + '<STR_LIT>',<EOL>default=consts.DEFAULT_ADDRESS)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + str(consts.DEFAULT_PORT) + '<STR_LIT:)>',<EOL>type=int,<EOL>default=consts.DEFAULT_PORT)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>'<EOL>+ str(consts.DEFAULT_BASE_URL) + '<STR_LIT:)>',<EOL>default=consts.DEFAULT_BASE_URL)<EOL>return parser<EOL>


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_cluster_role_topologies(cluster, role))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topologies under a cluster submitted by a role


qs = _all_metric_queries()<EOL>return dict(zip(qs[<NUM_LIT:0>], qs[<NUM_LIT:1>]) + zip(qs[<NUM_LIT:2>], qs[<NUM_LIT:3>]))<EOL>


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_cluster_topologies(cluster))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topologies under a cluster


def get_cluster_role_env_topologies(cluster, role, env):

instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topologies under a cluster submitted by a role under env


def fetch(self, cluster, metric, topology, component, instance, timerange, envirn=None):

pass<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param envirn:
:return:


def fetch_max(self, cluster, metric, topology, component, instance, timerange, envirn=None):

pass<EOL>


@tornado.gen.coroutine<EOL>def get_metrics(cluster, environment, topology, timerange, query, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environment,<EOL>topology=topology,<EOL>starttime=timerange[<NUM_LIT:0>],<EOL>endtime=timerange[<NUM_LIT:1>],<EOL>query=query)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(METRICS_QUERY_URL_FMT), params<EOL>)<EOL>logging.info("<STR_LIT>", request_url)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the metrics for a topology from tracker
:param cluster:
:param environment:
:param topology:
:param timerange:
:param query:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_component_exceptionsummary(cluster, environ, topology, component, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(EXCEPTION_SUMMARY_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get summary of exception for a component
:param cluster:
:param environ:
:param topology:
:param component:
:param role:
:return:


def get_container_file_download_url(cluster, environ, topology, container,<EOL>path, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(FILE_DOWNLOAD_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>return request_url<EOL>

:param cluster:
:param environ:
:param topology:
:param container:
:param path:
:param role:
:return:


@tornado.gen.coroutine<EOL><INDENT>def fetch_backpressure(self, cluster, metric, topology, component, instance,timerange, is_max, environ=None):<DEDENT>

instances = yield get_instances(cluster, environ, topology)<EOL>if component != "<STR_LIT:*>":<EOL><INDENT>filtered_inst = [instance for instance in instances if instance.split("<STR_LIT:_>")[<NUM_LIT:2>] == component]<EOL><DEDENT>else:<EOL><INDENT>filtered_inst = instances<EOL><DEDENT>futures_dict = {}<EOL>for inst in filtered_inst:<EOL><INDENT>query = queries.get(metric).format(inst)<EOL>futures_dict[inst] = get_metrics(cluster, environ, topology, timerange, query)<EOL><DEDENT>res = yield futures_dict<EOL>if not is_max:<EOL><INDENT>timelines = []<EOL>for key in res:<EOL><INDENT>result = res[key]<EOL>if len(result["<STR_LIT>"]) > <NUM_LIT:0>:<EOL><INDENT>result["<STR_LIT>"][<NUM_LIT:0>]["<STR_LIT>"] = key<EOL><DEDENT>timelines.extend(result["<STR_LIT>"])<EOL><DEDENT>result = self.get_metric_response(timerange, timelines, is_max)<EOL><DEDENT>else:<EOL><INDENT>data = self.compute_max(res.values())<EOL>result = self.get_metric_response(timerange, data, is_max)<EOL><DEDENT>raise tornado.gen.Return(result)<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param isMax:
:param environ:
:return:


@tornado.gen.coroutine<EOL>def get_comps(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(LOGICALPLAN_URL_FMT), params)<EOL>lplan = yield fetch_url_as_json(request_url)<EOL>comps = lplan['<STR_LIT>'].keys() + lplan['<STR_LIT>'].keys()<EOL>raise tornado.gen.Return(comps)<EOL>

Get the list of component names for the topology from Heron Nest
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_comp_metrics_timeline(cluster, environ, topology, component,<EOL>instances, metricnames, time_range, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(METRICS_TIMELINE_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>for metric_name in metricnames:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))<EOL><DEDENT>for instance in instances:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>request_url, dict(starttime=time_range[<NUM_LIT:0>], endtime=time_range[<NUM_LIT:1>]))<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the minute-by-minute metrics for all instances of a topology from tracker
:param cluster:
:param environ:
:param topology:
:param component:
:param instances:
:param metricnames:   dict of display name to cuckoo name
:param time_range:    2-tuple consisting of start and end of range
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_comp_metrics(cluster, environ, topology, component,<EOL>instances, metricnames, time_range, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(METRICS_URL_FMT), params)<EOL>for metric_name in metricnames:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))<EOL><DEDENT>for instance in instances:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))<EOL><DEDENT>request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[<NUM_LIT:1>]))<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the metrics for all the instances of a topology from Heron Nest
:param cluster:
:param environ:
:param topology:
:param component:
:param instances:
:param metricnames:   dict of display name to cuckoo name
:param time_range:    2-tuple consisting of start and end of range
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_filestats(cluster, environ, topology, container, path, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(FILESTATS_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


@tornado.gen.coroutine<EOL>def run_instance_jmap(cluster, environ, topology, instance, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>instance=instance)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(JMAP_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

:param cluster:
:param environ:
:param topology:
:param instance:
:param role:
:return:


return _get_topologies(cluster, role=role, env=env)<EOL>

Get the list of topologies given a cluster submitted by a given role under a given environment
:param cluster:
:param role:
:param env:
:return:


@tornado.gen.coroutine<EOL>def get_container_file_data(cluster, environ, topology, container,<EOL>path, offset, length, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path,<EOL>offset=offset,<EOL>length=length)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(FILE_DATA_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

:param cluster:
:param environ:
:param topology:
:param container:
:param path:
:param offset:
:param length:
:param role:
:return:


@tornado.gen.coroutine<EOL><INDENT>def fetch(self, cluster, metric, topology, component, instance, timerange, environ=None):<DEDENT>

components = [component] if component != "<STR_LIT:*>" else (yield get_comps(cluster, environ, topology))<EOL>futures = []<EOL>for comp in components:<EOL><INDENT>query = self.get_query(metric, comp, instance)<EOL>future = get_metrics(cluster, environ, topology, timerange, query)<EOL>futures.append(future)<EOL><DEDENT>results = yield futures<EOL>timelines = []<EOL>for result in results:<EOL><INDENT>timelines.extend(result["<STR_LIT>"])<EOL><DEDENT>result = self.get_metric_response(timerange, timelines, False)<EOL>raise tornado.gen.Return(result)<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param environ:
:return:


@tornado.gen.coroutine<EOL>def get_execution_state(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(EXECUTION_STATE_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the execution state of a topology in a cluster
:param cluster:
:param environ:
:param topology:
:param role:
:return:


return options.tracker_url<EOL>

Get the endpoint for heron tracker
:return:


request_url = create_url(TOPOLOGIES_URL_FMT)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the list of topologies given a data center from heron tracker
:return:


@tornado.gen.coroutine<EOL><INDENT>def fetch_max(self, cluster, metric, topology, component, instance, timerange, environ=None):<DEDENT>

components = [component] if component != "<STR_LIT:*>" else (yield get_comps(cluster, environ, topology))<EOL>result = {}<EOL>futures = []<EOL>for comp in components:<EOL><INDENT>query = self.get_query(metric, comp, instance)<EOL>max_query = "<STR_LIT>" % query<EOL>future = get_metrics(cluster, environ, topology, timerange, max_query)<EOL>futures.append(future)<EOL><DEDENT>results = yield futures<EOL>data = self.compute_max(results)<EOL>result = self.get_metric_response(timerange, data, True)<EOL>raise tornado.gen.Return(result)<EOL>


@tornado.gen.coroutine<EOL>def get_scheduler_location(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(SCHEDULER_LOCATION_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the scheduler location of a topology in a cluster from tracker
:param cluster:
:param environ:
:param topology:
:param role:
:return:


def get_metric_response(self, timerange, data, isMax):

:param timerange:
:param data:
:param isMax:
:return:


return fmt % get_tracker_endpoint()<EOL>

Given an URL format, substitute with tracker service endpoint
:param fmt:
:return:


@tornado.gen.coroutine<EOL>def get_instance_mem_histogram(cluster, environ, topology, instance, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>instance=instance)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(HISTOGRAM_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


@tornado.gen.coroutine<EOL>def get_topology_info(cluster, environ, topology, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(INFO_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

:param cluster:
:param environ:
:param topology:
:param role:
:return:


request_url = create_url(TOPOLOGIES_STATS_URL_FMT)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the list of topologies and their states
:return:


if len(multi_ts) > <NUM_LIT:0> and len(multi_ts[<NUM_LIT:0>]["<STR_LIT>"]) > <NUM_LIT:0>:<EOL><INDENT>keys = multi_ts[<NUM_LIT:0>]["<STR_LIT>"][<NUM_LIT:0>]["<STR_LIT:data>"].keys()<EOL>timelines = ([res["<STR_LIT>"][<NUM_LIT:0>]["<STR_LIT:data>"][key] for key in keys] for res in multi_ts)<EOL>values = (max(v) for v in zip(*timelines))<EOL>return dict(zip(keys, values))<EOL><DEDENT>return {}<EOL>


return _get_topologies(cluster)<EOL>

Get the list of topologies given a cluster
:param cluster:
:return:


<EOL>cli_conf_file = os.path.join(config_path, CLIENT_YAML)<EOL>if not os.path.isfile(cli_conf_file):<EOL><INDENT>return True<EOL><DEDENT>client_confs = {}<EOL>with open(cli_conf_file, '<STR_LIT:r>') as conf_file:<EOL><INDENT>client_confs = yaml.load(conf_file)<EOL>if not client_confs:<EOL><INDENT>return True<EOL><DEDENT>role_present = True if len(cluster_role_env[<NUM_LIT:1>]) > <NUM_LIT:0> else False<EOL>if ROLE_REQUIRED in client_confs and client_confs[ROLE_REQUIRED] and not role_present:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (cluster_role_env, ROLE_REQUIRED, cli_conf_file))<EOL><DEDENT>environ_present = True if len(cluster_role_env[<NUM_LIT:2>]) > <NUM_LIT:0> else False<EOL>if ENV_REQUIRED in client_confs and client_confs[ENV_REQUIRED] and not environ_present:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (cluster_role_env, ENV_REQUIRED, cli_conf_file))<EOL><DEDENT><DEDENT>return True<EOL>

Check cluster/[role]/[environ], if they are required


return '<STR_LIT::>'.join(map(normalized_class_path, jars))<EOL>

Get the normalized class path of all jars


conf_path = os.path.join(get_heron_dir(), CONF_DIR)<EOL>return conf_path<EOL>

This will provide heron conf directory from .pex file.
:return: absolute path of heron conf directory


java_home = os.environ.get("<STR_LIT>")<EOL>return os.path.join(java_home, BIN_DIR, "<STR_LIT>")<EOL>


parts = cluster_role_env.split('<STR_LIT:/>')[:<NUM_LIT:3>]<EOL>if len(parts) == <NUM_LIT:3>:<EOL><INDENT>return (parts[<NUM_LIT:0>], parts[<NUM_LIT:1>], parts[<NUM_LIT:2>])<EOL><DEDENT>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>return (parts[<NUM_LIT:0>], parts[<NUM_LIT:1>], "<STR_LIT>")<EOL><DEDENT>if len(parts) == <NUM_LIT:1>:<EOL><INDENT>return (parts[<NUM_LIT:0>], "<STR_LIT>", "<STR_LIT>")<EOL><DEDENT>return ("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>

Parse cluster/[role]/[environ], supply empty string, if not provided


if zipped_pex:<EOL><INDENT>release_file = get_zipped_heron_release_file()<EOL><DEDENT>else:<EOL><INDENT>release_file = get_heron_release_file()<EOL><DEDENT>with open(release_file) as release_info:<EOL><INDENT>release_map = yaml.load(release_info)<EOL>release_items = sorted(release_map.items(), key=lambda tup: tup[<NUM_LIT:0>])<EOL>for key, value in release_items:<EOL><INDENT>print("<STR_LIT>" % (key, value))<EOL><DEDENT><DEDENT>

Print build_info from release.yaml

    :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'.


bin_path = os.path.join(get_heron_dir(), BIN_DIR)<EOL>return bin_path<EOL>

This will provide heron bin directory from .pex file.
:return: absolute path of heron lib directory


release_file = get_heron_release_file()<EOL>if not os.path.isfile(release_file):<EOL><INDENT>Log.error("<STR_LIT>" % release_file)<EOL>return False<EOL><DEDENT>return True<EOL>


go_above_dirs = <NUM_LIT:9><EOL>path = "<STR_LIT:/>".join(os.path.realpath(__file__).split('<STR_LIT:/>')[:-go_above_dirs])<EOL>return normalized_class_path(path)<EOL>

This will extract heron directory from .pex file.

For example,
when __file__ is '/Users/heron-user/bin/heron/heron/tools/common/src/python/utils/config.pyc', and
its real path is '/Users/heron-user/.heron/bin/heron/tools/common/src/python/utils/config.pyc',
the internal variable ``path`` would be '/Users/heron-user/.heron', which is the heron directory

This means the variable `go_above_dirs` below is 9.

:return: root location of the .pex file


<EOL>subparsers_actions = [action for action in parser._actions<EOL>if isinstance(action, argparse._SubParsersAction)]<EOL>for subparsers_action in subparsers_actions:<EOL><INDENT>for choice, subparser in subparsers_action.choices.items():<EOL><INDENT>if choice == command:<EOL><INDENT>return subparser<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

Retrieve the given subparser from parser


index = <NUM_LIT:0><EOL>found = False<EOL>for lelem in command_args:<EOL><INDENT>if lelem == '<STR_LIT>' and not found:<EOL><INDENT>break<EOL><DEDENT>if lelem == param:<EOL><INDENT>found = True<EOL>break<EOL><DEDENT>index = index + <NUM_LIT:1><EOL><DEDENT>if found:<EOL><INDENT>command_args.insert(index + <NUM_LIT:1>, '<STR_LIT:True>')<EOL><DEDENT>return command_args<EOL>

:param param:
:param command_args:
:return:


return os.path.join(default_config_path, cluster)<EOL>

This will provide heron cluster config directory, if config path is default
:return: absolute path of heron cluster conf directory


heron_lib_dir = get_heron_lib_dir()<EOL>heron_libs = [os.path.join(heron_lib_dir, f) for f in local_jars]<EOL>return heron_libs<EOL>

Get all the heron lib jars with the absolute paths


<EOL>if path.endswith('<STR_LIT:*>'):<EOL><INDENT>Log.debug('<STR_LIT>', path[:-<NUM_LIT:1>])<EOL>if os.path.isdir(path[:-<NUM_LIT:1>]):<EOL><INDENT>return True<EOL><DEDENT>return False<EOL><DEDENT>Log.debug('<STR_LIT>', path)<EOL>if os.path.isdir(path):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>Log.debug('<STR_LIT>', path)<EOL>if os.path.isfile(path):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Check if an entry in the class path exists as either a directory or a file


self.locations = state_locations<EOL>self.validate_state_locations()<EOL>


def get_pplan(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.pplan_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>pplan_path = self.get_pplan_path(topologyName)<EOL>with open(pplan_path) as f:<EOL><INDENT>data = f.read()<EOL>pplan = PhysicalPlan()<EOL>pplan.ParseFromString(data)<EOL>return pplan<EOL><DEDENT><DEDENT>


def get_scheduler_location(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.scheduler_location_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>scheduler_location_path = self.get_scheduler_location_path(topologyName)<EOL>with open(scheduler_location_path) as f:<EOL><INDENT>data = f.read()<EOL>scheduler_location = SchedulerLocation()<EOL>scheduler_location.ParseFromString(data)<EOL>return scheduler_location<EOL><DEDENT><DEDENT>


self.monitoring_thread_stop_signal = False<EOL>self.monitoring_thread.start()<EOL>


def create_topology(self, topologyName, topology):

pass<EOL>

Create path is currently not supported in file based state manager.


def get_packing_plan(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.packing_plan_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>packing_plan_path = self.get_packing_plan_path(topologyName)<EOL>with open(packing_plan_path) as f:<EOL><INDENT>data = f.read()<EOL>packing_plan = PackingPlan()<EOL>packing_plan.ParseFromString(data)<EOL><DEDENT><DEDENT>


def create_pplan(self, topologyName, pplan):

pass<EOL>


def get_tmaster(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.tmaster_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>tmaster_path = self.get_tmaster_path(topologyName)<EOL>with open(tmaster_path) as f:<EOL><INDENT>data = f.read()<EOL>tmaster = TMasterLocation()<EOL>tmaster.ParseFromString(data)<EOL>return tmaster<EOL><DEDENT><DEDENT>


pass<EOL>

Delete path is currently not supported in file based state manager.


def get_execution_state(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.execution_state_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>execution_state_path = self.get_execution_state_path(topologyName)<EOL>with open(execution_state_path) as f:<EOL><INDENT>data = f.read()<EOL>executionState = ExecutionState()<EOL>executionState.ParseFromString(data)<EOL>return executionState<EOL><DEDENT><DEDENT>


def create_execution_state(self, topologyName, executionState):

pass<EOL>


pass<EOL>


log_format = "<STR_LIT>"<EOL>date_format = '<STR_LIT>'<EOL>logging.basicConfig(format=log_format, datefmt=date_format)<EOL>Log.setLevel(level)<EOL>if logfile is not None:<EOL><INDENT>fh = logging.FileHandler(logfile)<EOL>fh.setFormatter(logging.Formatter(log_format))<EOL>Log.addHandler(fh)<EOL><DEDENT>


state_managers = []<EOL>try:<EOL><INDENT>state_managers.extend(get_all_zk_state_managers(conf))<EOL>state_managers.extend(get_all_file_state_managers(conf))<EOL>return state_managers<EOL><DEDENT>except Exception as ex:<EOL><INDENT>LOG.error("<STR_LIT>")<EOL>raise ex<EOL><DEDENT>

@param conf - An instance of Config class
Reads the config for requested state managers.
Instantiates them, start and then return them.


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_pplan_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


return KazooClient(hostportlist)<EOL>

For Unit testing, replace this method to not
Actually return a client


if not pplan or not pplan.IsInitialized():<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>path = self.get_pplan_path(topologyName)<EOL>LOG.info("<STR_LIT>".format(<EOL>topologyName, path))<EOL>pplanString = pplan.SerializeToString()<EOL>try:<EOL><INDENT>self.client.create(path, value=pplanString, makepath=True)<EOL>return True<EOL><DEDENT>except NoNodeError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except NodeExistsError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except ZookeeperError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except Exception:<EOL><INDENT>raise<EOL><DEDENT>


def _get_topology_with_watch(self, topologyName, callback, isWatching):

path = self.get_topology_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_topology(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>topology = Topology()<EOL>topology.ParseFromString(data)<EOL>callback(topology)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>

Helper function to get pplan with
a callback. The future watch is placed
only if isWatching is True.


def _get_execution_state_with_watch(self, topologyName, callback, isWatching):

path = self.get_execution_state_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_execution_state(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>executionState = ExecutionState()<EOL>executionState.ParseFromString(data)<EOL>callback(executionState)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>

Helper function to get execution state with
a callback. The future watch is placed
only if isWatching is True.


if not topology or not topology.IsInitialized():<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>path = self.get_topology_path(topologyName)<EOL>LOG.info("<STR_LIT>".format(<EOL>topologyName, path))<EOL>topologyString = topology.SerializeToString()<EOL>try:<EOL><INDENT>self.client.create(path, value=topologyString, makepath=True)<EOL>return True<EOL><DEDENT>except NoNodeError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except NodeExistsError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except ZookeeperError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except Exception:<EOL><INDENT>raise<EOL><DEDENT>


if self.is_host_port_reachable():<EOL><INDENT>self.client = self._kazoo_client(_makehostportlist(self.hostportlist))<EOL><DEDENT>else:<EOL><INDENT>localhostports = self.establish_ssh_tunnel()<EOL>self.client = self._kazoo_client(_makehostportlist(localhostports))<EOL><DEDENT>self.client.start()<EOL>def on_connection_change(state):<EOL><INDENT>"""<STR_LIT>"""<EOL>LOG.info("<STR_LIT>" + state)<EOL><DEDENT>self.client.add_listener(on_connection_change)<EOL>


def get_topology(self, topologyName, callback=None):

isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_topology_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def __replace(config, wildcards, config_file):

for config_key in config:<EOL><INDENT>config_value = config[config_key]<EOL>original_value = config_value<EOL>if isinstance(config_value, str):<EOL><INDENT>for token in wildcards:<EOL><INDENT>if wildcards[token]:<EOL><INDENT>config_value = config_value.replace(token, wildcards[token])<EOL><DEDENT><DEDENT>found = re.findall(r'<STR_LIT>', config_value)<EOL>if found:<EOL><INDENT>raise ValueError("<STR_LIT>" %<EOL>(config_key, original_value, config_file, "<STR_LIT:U+002CU+0020>".join(found)))<EOL><DEDENT>config[config_key] = config_value<EOL><DEDENT><DEDENT>return config<EOL>

For each kvp in config, do wildcard substitution on the values


@abc.abstractmethod<EOL><INDENT>def get_packing_plan(self, topologyName, callback=None):<DEDENT>

pass<EOL>

Gets the packing_plan for the topology.
If the callback is provided,
sets watch on the path and calls the callback
with the new packing_plan.


localportlist = []<EOL>for (host, port) in self.hostportlist:<EOL><INDENT>localport = self.pick_unused_port()<EOL>self.tunnel.append(subprocess.Popen(<EOL>('<STR_LIT>', self.tunnelhost, '<STR_LIT>' % (localport, host, port))))<EOL>localportlist.append(('<STR_LIT:127.0.0.1>', localport))<EOL><DEDENT>return localportlist<EOL>

Establish an ssh tunnel for each local host and port
that can be used to communicate with the state host.


pass<EOL>

If the state manager had connected to a remote server, it would need to stop as well.


self.delete_pplan(topologyName)<EOL>self.delete_execution_state(topologyName)<EOL>self.delete_topology(topologyName)<EOL>

Removes the topology entry from:
1. topologies list,
2. pplan,
3. execution_state, and


self.__tunnelhost = newTunnelHost<EOL>

Setter for the tunnelhost to create the tunnel if host is not accessible


self.__hostport = newRootPath<EOL>

Setter for the path where the heron states are stored.


pass<EOL>

If the state manager needs to connect to a remote host.


def status_finish(ret):<EOL><INDENT>self.set_status(ret)<EOL>self.finish()<EOL><DEDENT>def kill_parent():<EOL><INDENT>status_finish(<NUM_LIT:200>)<EOL>logger.info("<STR_LIT>")<EOL>os.killpg(os.getppid(), signal.SIGTERM)<EOL><DEDENT>logger = logging.getLogger(__file__)<EOL>logger.info("<STR_LIT>")<EOL>data = dict(urlparse.parse_qsl(self.request.body))<EOL>sharedSecret = data.get('<STR_LIT>')<EOL>if sharedSecret != options.secret:<EOL><INDENT>status_finish(<NUM_LIT>)<EOL>return<EOL><DEDENT>instanceId = data.get('<STR_LIT>')<EOL>if instanceId:<EOL><INDENT>filepath = instanceId + '<STR_LIT>'<EOL>if os.path.isfile(filepath): <EOL><INDENT>if instanceId.startswith('<STR_LIT>'): <EOL><INDENT>kill_parent()<EOL><DEDENT>else: <EOL><INDENT>fh = open(filepath)<EOL>firstLine = int(fh.readline())<EOL>fh.close()<EOL>logger.info("<STR_LIT>" + instanceId + "<STR_LIT:U+0020>" + str(firstLine))<EOL>os.kill(firstLine, signal.SIGTERM)<EOL>status_finish(<NUM_LIT:200>)<EOL><DEDENT><DEDENT>else: <EOL><INDENT>logger.info(filepath + "<STR_LIT>")<EOL>status_finish(<NUM_LIT>)<EOL><DEDENT><DEDENT>else: <EOL><INDENT>kill_parent()<EOL><DEDENT>


logging.debug("<STR_LIT>", path)<EOL>self.connection_closed = False<EOL>self.set_header("<STR_LIT>", "<STR_LIT>")<EOL>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>if path is None or not os.path.isfile(path):<EOL><INDENT>self.write("<STR_LIT>" % path)<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>length = int(<NUM_LIT:4> * <NUM_LIT> * <NUM_LIT>)<EOL>offset = int(<NUM_LIT:0>)<EOL>while True:<EOL><INDENT>data = utils.read_chunk(path, offset=offset, length=length, escape_data=False)<EOL>if self.connection_closed or '<STR_LIT:data>' not in data or len(data['<STR_LIT:data>']) < length:<EOL><INDENT>break<EOL><DEDENT>offset += length<EOL>self.write(data['<STR_LIT:data>'])<EOL>self.flush()<EOL><DEDENT>if '<STR_LIT:data>' in data:<EOL><INDENT>self.write(data['<STR_LIT:data>'])<EOL><DEDENT>self.finish()<EOL>


<EOL>self.connection_closed = True<EOL>


body = utils.str_cmd(['<STR_LIT>', pid], None, None)<EOL>self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(body))<EOL>self.finish()<EOL>


t = Template(utils.get_asset("<STR_LIT>"))<EOL>if path is None:<EOL><INDENT>self.set_status(<NUM_LIT>)<EOL>self.write("<STR_LIT>")<EOL>self.finish()<EOL>return<EOL><DEDENT>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>args = dict(<EOL>filename=path,<EOL>jquery=utils.get_asset("<STR_LIT>"),<EOL>pailer=utils.get_asset("<STR_LIT>"),<EOL>css=utils.get_asset("<STR_LIT>"),<EOL>)<EOL>self.write(t.generate(**args))<EOL>self.finish()<EOL>


body = utils.str_cmd(['<STR_LIT>', '<STR_LIT>', pid], None, None)<EOL>self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(body))<EOL>self.finish()<EOL>


utils.str_cmd(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'], None, None)<EOL>body = utils.str_cmd(['<STR_LIT>', '<STR_LIT>',<EOL>str(pid)], None, None)<EOL>utils.str_cmd(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'], None, None)<EOL>self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(body))<EOL>self.finish()<EOL>


if path != "<STR_LIT:.>":<EOL><INDENT>listing = sorted(['<STR_LIT:..>'] + os.listdir(path))<EOL><DEDENT>else:<EOL><INDENT>listing = sorted(os.listdir(path))<EOL><DEDENT>return listing<EOL>

Returns the list of files and directories in a path.
Prepents a ".." (parent directory link) if path is not current dir.


command = '<STR_LIT>'.join(map(lambda x: '<STR_LIT:U+0020>'.join(x), cmd_list))<EOL>chained_proc = functools.reduce(pipe, [None] + cmd_list)<EOL>stdout_builder = proc.async_stdout_builder(chained_proc)<EOL>chained_proc.wait()<EOL>return {<EOL>'<STR_LIT>': command,<EOL>'<STR_LIT>': stdout_builder.result()<EOL>}<EOL>

Feed output of one command to the next and return final output
Returns string output of chained application of commands.


return pkgutil.get_data("<STR_LIT>", os.path.join("<STR_LIT>", asset_name))<EOL>


def read_chunk(filename, offset=-<NUM_LIT:1>, length=-<NUM_LIT:1>, escape_data=False):

try:<EOL><INDENT>length = int(length)<EOL>offset = int(offset)<EOL><DEDENT>except ValueError:<EOL><INDENT>return {}<EOL><DEDENT>if not os.path.isfile(filename):<EOL><INDENT>return {}<EOL><DEDENT>try:<EOL><INDENT>fstat = os.stat(filename)<EOL><DEDENT>except Exception:<EOL><INDENT>return {}<EOL><DEDENT>if offset == -<NUM_LIT:1>:<EOL><INDENT>offset = fstat.st_size<EOL><DEDENT>if length == -<NUM_LIT:1>:<EOL><INDENT>length = fstat.st_size - offset<EOL><DEDENT>with open(filename, "<STR_LIT:r>") as fp:<EOL><INDENT>fp.seek(offset)<EOL>try:<EOL><INDENT>data = fp.read(length)<EOL><DEDENT>except IOError:<EOL><INDENT>return {}<EOL><DEDENT><DEDENT>if data:<EOL><INDENT>data = _escape_data(data) if escape_data else data<EOL>return dict(offset=offset, length=len(data), data=data)<EOL><DEDENT>return dict(offset=offset, length=<NUM_LIT:0>)<EOL>

Read a chunk of a file from an offset upto the length.


try:<EOL><INDENT>pwent = pwd.getpwuid(sres.st_uid)<EOL>user = pwent.pw_name<EOL><DEDENT>except KeyError:<EOL><INDENT>user = sres.st_uid<EOL><DEDENT>try:<EOL><INDENT>grent = grp.getgrgid(sres.st_gid)<EOL>group = grent.gr_name<EOL><DEDENT>except KeyError:<EOL><INDENT>group = sres.st_gid<EOL><DEDENT>return '<STR_LIT>' % (<EOL>format_mode(sres),<EOL>sres.st_nlink,<EOL>user,<EOL>group,<EOL>sres.st_size,<EOL>format_mtime(sres.st_mtime),<EOL>)<EOL>

Prefix to a filename in the directory listing. This is to make the
listing similar to an output of "ls -alh".


return not path.startswith("<STR_LIT:/>") and "<STR_LIT:..>" not in path<EOL>

file path should be a relative path without ".." in it
:param path: file path
:return: true if the path is relative and doesn't contain ".."


reqid = REQID.generate()<EOL>message = mock_protobuf.get_mock_register_response()<EOL>pkt = convert_to_incoming_packet(reqid, message)<EOL>return pkt, reqid, message<EOL>

Returns a tuple of mock (IncomingPacket, REQID, RegisterResponse message)


raw = OutgoingPacket.create_packet(reqid, message).raw<EOL>dispatcher = MockDispatcher()<EOL>dispatcher.prepare_with_raw(raw)<EOL>packet = IncomingPacket()<EOL>packet.read(dispatcher)<EOL>packet.data = packet.data<EOL>return packet<EOL>

Convert (reqid, message) pair to IncomingPacket object


self.eagain_test = True<EOL>

prepare so that EAGAIN error is raised when recv() is called


return len(buf)<EOL>

mock sends the content of a given buffer


self.fatal_error_test = True<EOL>

prepare so that RuntimeError is raised when recv() is called


pkt = get_mock_requst_packets(is_message=False)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>self.to_be_received = pkt.header<EOL>

a packet with just a header (incomplete packet) will be prepared in the recv buffer


if self.fatal_error_test:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>elif self.eagain_test:<EOL><INDENT>raise socket.error(socket.errno.EAGAIN, "<STR_LIT>")<EOL><DEDENT>ret = self.to_be_received[:numbytes]<EOL>self.to_be_received = self.to_be_received[numbytes:]<EOL>return ret<EOL>


spout_1 = mock_protobuf.get_mock_spout(component=mock_protobuf.get_mock_component(name="<STR_LIT>"))<EOL>bolt_1 = mock_protobuf.get_mock_bolt(component=mock_protobuf.get_mock_component(name="<STR_LIT>"))<EOL>bolt_2 = mock_protobuf.get_mock_bolt(component=mock_protobuf.get_mock_component(name="<STR_LIT>"))<EOL>topology = mock_protobuf.get_mock_topology(spouts=[spout_1], bolts=[bolt_1, bolt_2])<EOL>instance_ids = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>task_ids = [<NUM_LIT:100>, <NUM_LIT:200>, <NUM_LIT>]<EOL>comp_indexes = [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]<EOL>comp_names = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>instances = []<EOL>for i_id, t_id, c_i, c_name in zip(instance_ids, task_ids, comp_indexes, comp_names):<EOL><INDENT>info = mock_protobuf.get_mock_instance_info(task_id=t_id,<EOL>component_index=c_i,<EOL>component_name=c_name)<EOL>instance = mock_protobuf.get_mock_instance(instance_id=i_id, info=info)<EOL>instances.append(instance)<EOL><DEDENT>pplan = mock_protobuf.get_mock_pplan(topology=topology, instances=instances)<EOL>keys = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>zipped = zip(instance_ids, task_ids, comp_indexes, comp_names)<EOL>return pplan, [dict(zip(keys, z)) for z in zipped]<EOL>

Returns a legitimate looking physical plan

    This topology has 1 spout and 2 bolts. Currently no input/output streams.
    There is only one stream manager.

    [Instance 1: spout1]
      - instance_id = "instance1"
      - task_id = 100
      - component_index = 0
      - component_name = "spout1"

    [Instance 2: bolt1]
      - instance_id = "instance2"
      - task_id = 200
      - component_index = 0
      - component_name = "bolt1"

    [instance 3: bolt2]
      - instance_id = "instance3"
      - task_id = 300
      - component_index = 0
      - component_name = "bolt2"

    :returns: PhysicalPlan message and a list of dictionaries for each instance containing
              (instance_id, task_id, comp_index, comp_name)


data_tuple = tuple_pb2.HeronDataTuple()<EOL>data_tuple.key = <NUM_LIT:0><EOL>tuple_size_in_bytes = <NUM_LIT:0><EOL>for obj in lst:<EOL><INDENT>serialized = serializer.serialize(obj)<EOL>data_tuple.values.append(serialized)<EOL>tuple_size_in_bytes += len(serialized)<EOL><DEDENT>return data_tuple, tuple_size_in_bytes<EOL>

Make HeronDataTuple from a list of objects


def get_mock_bolt(component=get_mock_component(), inputs=[], outputs=[]):

bolt = topology_pb2.Bolt()<EOL>bolt.comp.CopyFrom(component)<EOL>for i in inputs:<EOL><INDENT>added = bolt.inputs.add()<EOL>added.CopyFrom(i)<EOL><DEDENT>for o in outputs:<EOL><INDENT>added = bolt.outputs.add()<EOL>added.CopyFrom(o)<EOL><DEDENT>return bolt<EOL>

Returns a mock protobuf Bolt object from topology_pb2


if config_dict is None:<EOL><INDENT>return topology_pb2.Config()<EOL><DEDENT>proto_config = topology_pb2.Config()<EOL>config_serializer = PythonSerializer()<EOL>assert isinstance(config_dict, dict)<EOL>for key, value in config_dict.items():<EOL><INDENT>if isinstance(value, bool):<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.value = "<STR_LIT:true>" if value else "<STR_LIT:false>"<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT>elif isinstance(value, (str, int, float)):<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.value = str(value)<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.serialized_value = config_serializer.serialize(value)<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT><DEDENT>return proto_config<EOL>

Returns a protobuf Config object from topology_pb2


<EOL>builder = lambda: physical_plan_pb2.PhysicalPlan()<EOL>typename = builder().DESCRIPTOR.full_name<EOL>return builder, typename<EOL>

Returns a PhysicalPlan builder callable and typename 'PhysicalPlan


<EOL>mock_message = stmgr_pb2.NewInstanceAssignmentMessage()<EOL>mock_message.pplan.MergeFrom(pplan)<EOL>return mock_message<EOL>

Returns a mock protobuf NewInstanceAssignmentMessage object from stmgr_pb2


component = topology_pb2.Component()<EOL>component.name = name<EOL>component.spec = topology_pb2.ComponentObjectSpec.Value("<STR_LIT>")<EOL>component.class_name = python_cls<EOL>component.config.CopyFrom(config)<EOL>return component<EOL>

Returns a mock protobuf Component object from topology_pb2


stream_id = topology_pb2.StreamId()<EOL>stream_id.id = id<EOL>stream_id.component_name = component_name<EOL>return stream_id<EOL>

Returns a mock protobuf StreamId from topology_pb2


pplans_lst = []<EOL>for i in range(<NUM_LIT:10>):<EOL><INDENT>_id = "<STR_LIT>" + str(i)<EOL>pplan = get_mock_pplan(stmgrs=[get_mock_stmgr(id=_id)])<EOL>pplans_lst.append(pplan)<EOL><DEDENT>return pplans_lst<EOL>

Returns a list of 10 PhysicalPlan objects, differing just by stream manager id


spout = topology_pb2.Spout()<EOL>spout.comp.CopyFrom(component)<EOL>for out in outputs:<EOL><INDENT>added = spout.outputs.add()<EOL>added.CopyFrom(out)<EOL><DEDENT>return spout<EOL>

Returns a mock protobuf Spout object from topology_pb2


if level is None:<EOL><INDENT>_log_level = logging.INFO<EOL><DEDENT>else:<EOL><INDENT>if level == "<STR_LIT>" or level == "<STR_LIT>":<EOL><INDENT>_log_level = logging.DEBUG<EOL><DEDENT>elif level == "<STR_LIT:info>":<EOL><INDENT>_log_level = logging.INFO<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>_log_level = logging.WARNING<EOL><DEDENT>elif level == "<STR_LIT:error>":<EOL><INDENT>_log_level = logging.ERROR<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(level))<EOL><DEDENT><DEDENT>self.logger.log(_log_level, message)<EOL>

Log message, optionally providing a logging level

        It is compatible with StreamParse API.

        :type message: str
        :param message: the log message to send
        :type level: str
        :param level: the logging level,
                      one of: trace (=debug), debug, info, warn or error (default: info)


raise NotImplementedError()<EOL>


raise NotImplementedError()<EOL>

Should be called when a tuple was buffered into in_stream


raise NotImplementedError()<EOL>

Do the basic clean for Heron Instance

        Note that this method is not guaranteed to be invoked