if not query:<EOL><INDENT>return None<EOL><DEDENT>if query[<NUM_LIT:0>] == '<STR_LIT:(>':<EOL><INDENT>index = self.find_closing_braces(query)<EOL>if index != len(query) - <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return self.parse_query_string(query[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL><DEDENT><DEDENT>start_index = query.find("<STR_LIT:(>")<EOL>if start_index < <NUM_LIT:0>:<EOL><INDENT>try:<EOL><INDENT>constant = float(query)<EOL>return constant<EOL><DEDENT>except ValueError:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>token = query[:start_index]<EOL>if token not in self.operators:<EOL><INDENT>raise Exception("<STR_LIT>" + token)<EOL><DEDENT>rest_of_the_query = query[start_index:]<EOL>braces_end_index = self.find_closing_braces(rest_of_the_query)<EOL>if braces_end_index != len(rest_of_the_query) - <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>parts = self.get_sub_parts(rest_of_the_query[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>if token == "<STR_LIT>":<EOL><INDENT>return self.operators[token](parts)<EOL><DEDENT>children = []<EOL>for part in parts:<EOL><INDENT>children.append(self.parse_query_string(part))<EOL><DEDENT>node = self.operators[token](children)<EOL>return node<EOL>

Returns a parse tree for the query, each of the node is a
        subclass of Operator. This is both a lexical as well as syntax analyzer step.


config = {}<EOL>for kv in kvs:<EOL><INDENT>if kv.value:<EOL><INDENT>config[kv.key] = kv.value<EOL><DEDENT>elif kv.serialized_value:<EOL><INDENT>if topology_pb2.JAVA_SERIALIZED_VALUE == kv.type:<EOL><INDENT>jv = _convert_java_value(kv, include_non_primitives=include_non_primitives)<EOL>if jv is not None:<EOL><INDENT>config[kv.key] = jv<EOL><DEDENT><DEDENT>else:<EOL><INDENT>config[kv.key] = _raw_value(kv)<EOL><DEDENT><DEDENT><DEDENT>return config<EOL>


self.state_managers = statemanagerfactory.get_all_state_managers(self.config.statemgr_config)<EOL>try:<EOL><INDENT>for state_manager in self.state_managers:<EOL><INDENT>state_manager.start()<EOL><DEDENT><DEDENT>except Exception as ex:<EOL><INDENT>Log.error("<STR_LIT>" % ex)<EOL>traceback.print_exc()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>def on_topologies_watch(state_manager, topologies):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>")<EOL>Log.debug("<STR_LIT>" + str(topologies))<EOL>existingTopologies = self.getTopologiesForStateLocation(state_manager.name)<EOL>existingTopNames = map(lambda t: t.name, existingTopologies)<EOL>Log.debug("<STR_LIT>" + str(existingTopNames))<EOL>for name in existingTopNames:<EOL><INDENT>if name not in topologies:<EOL><INDENT>Log.info("<STR_LIT>",<EOL>name, state_manager.rootpath)<EOL>self.removeTopology(name, state_manager.name)<EOL><DEDENT><DEDENT>for name in topologies:<EOL><INDENT>if name not in existingTopNames:<EOL><INDENT>self.addNewTopology(state_manager, name)<EOL><DEDENT><DEDENT><DEDENT>for state_manager in self.state_managers:<EOL><INDENT>onTopologiesWatch = partial(on_topologies_watch, state_manager)<EOL>state_manager.get_topologies(onTopologiesWatch)<EOL><DEDENT>


def getTopologyByClusterRoleEnvironAndName(self, cluster, role, environ, topologyName):

topologies = list(filter(lambda t: t.name == topologyName<EOL>and t.cluster == cluster<EOL>and (not role or t.execution_state.role == role)<EOL>and t.environ == environ, self.topologies))<EOL>if not topologies or len(topologies) > <NUM_LIT:1>:<EOL><INDENT>if role is not None:<EOL><INDENT>raise Exception("<STR_LIT>".format(<EOL>cluster, role, environ, topologyName))<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>".format(<EOL>cluster, environ, topologyName))<EOL><DEDENT><DEDENT>return topologies[<NUM_LIT:0>]<EOL>

Find and return the topology given its cluster, environ, topology name, and
an optional role.
Raises exception if topology is not found, or more than one are found.


return filter(lambda t: t.state_manager_name == name, self.topologies)<EOL>

Returns all the topologies for a given state manager.


def addNewTopology(self, state_manager, topologyName):

topology = Topology(topologyName, state_manager.name)<EOL>Log.info("<STR_LIT>",<EOL>topologyName, state_manager.name)<EOL>self.topologies.append(topology)<EOL>topology.register_watch(self.setTopologyInfo)<EOL>def on_topology_pplan(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>" + topologyName)<EOL>topology.set_physical_plan(data)<EOL>if not data:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>def on_topology_packing_plan(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>" + topologyName)<EOL>topology.set_packing_plan(data)<EOL>if not data:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>def on_topology_execution_state(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>" + topologyName)<EOL>topology.set_execution_state(data)<EOL>if not data:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>def on_topology_tmaster(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>" + topologyName)<EOL>topology.set_tmaster(data)<EOL>if not data:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>def on_topology_scheduler_location(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info("<STR_LIT>" + topologyName)<EOL>topology.set_scheduler_location(data)<EOL>if not data:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>state_manager.get_pplan(topologyName, on_topology_pplan)<EOL>state_manager.get_packing_plan(topologyName, on_topology_packing_plan)<EOL>state_manager.get_execution_state(topologyName, on_topology_execution_state)<EOL>state_manager.get_tmaster(topologyName, on_topology_tmaster)<EOL>state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)<EOL>

Adds a topology in the local cache, and sets a watch
on any changes on the topology.


def removeTopology(self, topology_name, state_manager_name):

topologies = []<EOL>for top in self.topologies:<EOL><INDENT>if (top.name == topology_name and<EOL>top.state_manager_name == state_manager_name):<EOL><INDENT>if (topology_name, state_manager_name) in self.topologyInfos:<EOL><INDENT>self.topologyInfos.pop((topology_name, state_manager_name))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>topologies.append(top)<EOL><DEDENT><DEDENT>self.topologies = topologies<EOL>

Removes the topology from the local cache.


execution_state = topology.execution_state<EOL>executionState = {<EOL>"<STR_LIT>": execution_state.cluster,<EOL>"<STR_LIT>": execution_state.environ,<EOL>"<STR_LIT>": execution_state.role,<EOL>"<STR_LIT>": topology.name,<EOL>"<STR_LIT>": execution_state.submission_time,<EOL>"<STR_LIT>": execution_state.submission_user,<EOL>"<STR_LIT>": execution_state.release_state.release_username,<EOL>"<STR_LIT>": execution_state.release_state.release_tag,<EOL>"<STR_LIT>": execution_state.release_state.release_version,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": [],<EOL>}<EOL>for extra_link in self.config.extra_links:<EOL><INDENT>link = extra_link.copy()<EOL>link["<STR_LIT:url>"] = self.config.get_formatted_url(executionState,<EOL>link[EXTRA_LINK_FORMATTER_KEY])<EOL>executionState["<STR_LIT>"].append(link)<EOL><DEDENT>return executionState<EOL>

Returns the repesentation of execution state that will
be returned from Tracker.


execution_state = topology.execution_state<EOL>metadata = {<EOL>"<STR_LIT>": execution_state.cluster,<EOL>"<STR_LIT>": execution_state.environ,<EOL>"<STR_LIT>": execution_state.role,<EOL>"<STR_LIT>": topology.name,<EOL>"<STR_LIT>": execution_state.submission_time,<EOL>"<STR_LIT>": execution_state.submission_user,<EOL>"<STR_LIT>": execution_state.release_state.release_username,<EOL>"<STR_LIT>": execution_state.release_state.release_tag,<EOL>"<STR_LIT>": execution_state.release_state.release_version,<EOL>"<STR_LIT>": [],<EOL>}<EOL>for extra_link in self.config.extra_links:<EOL><INDENT>link = extra_link.copy()<EOL>link["<STR_LIT:url>"] = self.config.get_formatted_url(metadata,<EOL>link[EXTRA_LINK_FORMATTER_KEY])<EOL>metadata["<STR_LIT>"].append(link)<EOL><DEDENT>return metadata<EOL>

Returns metadata that will
be returned from Tracker.


schedulerLocation = {<EOL>"<STR_LIT:name>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>}<EOL>if topology.scheduler_location:<EOL><INDENT>schedulerLocation["<STR_LIT:name>"] = topology.scheduler_location.topology_name<EOL>schedulerLocation["<STR_LIT>"] = topology.scheduler_location.http_endpoint<EOL>schedulerLocation["<STR_LIT>"] =topology.scheduler_location.job_page_link[<NUM_LIT:0>]if len(topology.scheduler_location.job_page_link) > <NUM_LIT:0> else "<STR_LIT>"<EOL><DEDENT>return schedulerLocation<EOL>

Returns the representation of scheduler location that will
be returned from Tracker.


tmasterLocation = {<EOL>"<STR_LIT:name>": None,<EOL>"<STR_LIT:id>": None,<EOL>"<STR_LIT:host>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>}<EOL>if topology.tmaster:<EOL><INDENT>tmasterLocation["<STR_LIT:name>"] = topology.tmaster.topology_name<EOL>tmasterLocation["<STR_LIT:id>"] = topology.tmaster.topology_id<EOL>tmasterLocation["<STR_LIT:host>"] = topology.tmaster.host<EOL>tmasterLocation["<STR_LIT>"] = topology.tmaster.controller_port<EOL>tmasterLocation["<STR_LIT>"] = topology.tmaster.master_port<EOL>tmasterLocation["<STR_LIT>"] = topology.tmaster.stats_port<EOL><DEDENT>return tmasterLocation<EOL>

Returns the representation of tmaster that will
be returned from Tracker.


logicalPlan = {<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {},<EOL>}<EOL>for spout in topology.spouts():<EOL><INDENT>spoutName = spout.comp.name<EOL>spoutType = "<STR_LIT:default>"<EOL>spoutSource = "<STR_LIT>"<EOL>spoutVersion = "<STR_LIT>"<EOL>spoutConfigs = spout.comp.config.kvs<EOL>for kvs in spoutConfigs:<EOL><INDENT>if kvs.key == "<STR_LIT>":<EOL><INDENT>spoutType = javaobj.loads(kvs.serialized_value)<EOL><DEDENT>elif kvs.key == "<STR_LIT>":<EOL><INDENT>spoutSource = javaobj.loads(kvs.serialized_value)<EOL><DEDENT>elif kvs.key == "<STR_LIT>":<EOL><INDENT>spoutVersion = javaobj.loads(kvs.serialized_value)<EOL><DEDENT><DEDENT>spoutPlan = {<EOL>"<STR_LIT>": convert_pb_kvs(spoutConfigs, include_non_primitives=False),<EOL>"<STR_LIT:type>": spoutType,<EOL>"<STR_LIT:source>": spoutSource,<EOL>"<STR_LIT:version>": spoutVersion,<EOL>"<STR_LIT>": []<EOL>}<EOL>for outputStream in list(spout.outputs):<EOL><INDENT>spoutPlan["<STR_LIT>"].append({<EOL>"<STR_LIT>": outputStream.stream.id<EOL>})<EOL><DEDENT>logicalPlan["<STR_LIT>"][spoutName] = spoutPlan<EOL><DEDENT>for bolt in topology.bolts():<EOL><INDENT>boltName = bolt.comp.name<EOL>boltPlan = {<EOL>"<STR_LIT>": convert_pb_kvs(bolt.comp.config.kvs, include_non_primitives=False),<EOL>"<STR_LIT>": [],<EOL>"<STR_LIT>": []<EOL>}<EOL>for outputStream in list(bolt.outputs):<EOL><INDENT>boltPlan["<STR_LIT>"].append({<EOL>"<STR_LIT>": outputStream.stream.id<EOL>})<EOL><DEDENT>for inputStream in list(bolt.inputs):<EOL><INDENT>boltPlan["<STR_LIT>"].append({<EOL>"<STR_LIT>": inputStream.stream.id,<EOL>"<STR_LIT>": inputStream.stream.component_name,<EOL>"<STR_LIT>": topology_pb2.Grouping.Name(inputStream.gtype)<EOL>})<EOL><DEDENT>logicalPlan["<STR_LIT>"][boltName] = boltPlan<EOL><DEDENT>return logicalPlan<EOL>

Returns the representation of logical plan that will
be returned from Tracker.


physicalPlan = {<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": {}<EOL>}<EOL>if not topology.physical_plan:<EOL><INDENT>return physicalPlan<EOL><DEDENT>spouts = topology.spouts()<EOL>bolts = topology.bolts()<EOL>stmgrs = None<EOL>instances = None<EOL>stmgrs = list(topology.physical_plan.stmgrs)<EOL>instances = list(topology.physical_plan.instances)<EOL>if topology.physical_plan.topology.topology_config:<EOL><INDENT>physicalPlan["<STR_LIT>"] = convert_pb_kvs(topology.physical_plan.topology.topology_config.kvs)<EOL><DEDENT>for spout in spouts:<EOL><INDENT>spout_name = spout.comp.name<EOL>physicalPlan["<STR_LIT>"][spout_name] = []<EOL>if spout_name not in physicalPlan["<STR_LIT>"]:<EOL><INDENT>physicalPlan["<STR_LIT>"][spout_name] = {<EOL>"<STR_LIT>": convert_pb_kvs(spout.comp.config.kvs)<EOL>}<EOL><DEDENT><DEDENT>for bolt in bolts:<EOL><INDENT>bolt_name = bolt.comp.name<EOL>physicalPlan["<STR_LIT>"][bolt_name] = []<EOL>if bolt_name not in physicalPlan["<STR_LIT>"]:<EOL><INDENT>physicalPlan["<STR_LIT>"][bolt_name] = {<EOL>"<STR_LIT>": convert_pb_kvs(bolt.comp.config.kvs)<EOL>}<EOL><DEDENT><DEDENT>for stmgr in stmgrs:<EOL><INDENT>host = stmgr.host_name<EOL>cwd = stmgr.cwd<EOL>shell_port = stmgr.shell_port if stmgr.HasField("<STR_LIT>") else None<EOL>physicalPlan["<STR_LIT>"][stmgr.id] = {<EOL>"<STR_LIT:id>": stmgr.id,<EOL>"<STR_LIT:host>": host,<EOL>"<STR_LIT:port>": stmgr.data_port,<EOL>"<STR_LIT>": shell_port,<EOL>"<STR_LIT>": cwd,<EOL>"<STR_LIT>": stmgr.pid,<EOL>"<STR_LIT>": utils.make_shell_job_url(host, shell_port, cwd),<EOL>"<STR_LIT>": utils.make_shell_logfiles_url(host, shell_port, cwd),<EOL>"<STR_LIT>": []<EOL>}<EOL><DEDENT>instance_groups = collections.OrderedDict()<EOL>for instance in instances:<EOL><INDENT>instance_id = instance.instance_id<EOL>stmgrId = instance.stmgr_id<EOL>name = instance.info.component_name<EOL>stmgrInfo = physicalPlan["<STR_LIT>"][stmgrId]<EOL>host = stmgrInfo["<STR_LIT:host>"]<EOL>cwd = stmgrInfo["<STR_LIT>"]<EOL>shell_port = stmgrInfo["<STR_LIT>"]<EOL>group_name = instance_id.rsplit("<STR_LIT:_>", <NUM_LIT:2>)[<NUM_LIT:0>]<EOL>igroup = instance_groups.get(group_name, list())<EOL>igroup.append(instance_id)<EOL>instance_groups[group_name] = igroup<EOL>physicalPlan["<STR_LIT>"][instance_id] = {<EOL>"<STR_LIT:id>": instance_id,<EOL>"<STR_LIT:name>": name,<EOL>"<STR_LIT>": stmgrId,<EOL>"<STR_LIT>": utils.make_shell_logfiles_url(host, shell_port, cwd, instance.instance_id),<EOL>}<EOL>physicalPlan["<STR_LIT>"][stmgrId]["<STR_LIT>"].append(instance_id)<EOL>if name in physicalPlan["<STR_LIT>"]:<EOL><INDENT>physicalPlan["<STR_LIT>"][name].append(instance_id)<EOL><DEDENT>else:<EOL><INDENT>physicalPlan["<STR_LIT>"][name].append(instance_id)<EOL><DEDENT><DEDENT>physicalPlan["<STR_LIT>"] = instance_groups<EOL>return physicalPlan<EOL>

Returns the representation of physical plan that will
be returned from Tracker.


packingPlan = {<EOL>"<STR_LIT:id>": "<STR_LIT>",<EOL>"<STR_LIT>": []<EOL>}<EOL>if not topology.packing_plan:<EOL><INDENT>return packingPlan<EOL><DEDENT>container_plans = topology.packing_plan.container_plans<EOL>containers = []<EOL>for container_plan in container_plans:<EOL><INDENT>instances = []<EOL>for instance_plan in container_plan.instance_plans:<EOL><INDENT>instance_resources = {"<STR_LIT>": instance_plan.resource.cpu,<EOL>"<STR_LIT>": instance_plan.resource.ram,<EOL>"<STR_LIT>": instance_plan.resource.disk}<EOL>instance = {"<STR_LIT>" : instance_plan.component_name,<EOL>"<STR_LIT>" : instance_plan.task_id,<EOL>"<STR_LIT>": instance_plan.component_index,<EOL>"<STR_LIT>": instance_resources}<EOL>instances.append(instance)<EOL><DEDENT>required_resource = {"<STR_LIT>": container_plan.requiredResource.cpu,<EOL>"<STR_LIT>": container_plan.requiredResource.ram,<EOL>"<STR_LIT>": container_plan.requiredResource.disk}<EOL>scheduled_resource = {}<EOL>if container_plan.scheduledResource:<EOL><INDENT>scheduled_resource = {"<STR_LIT>": container_plan.scheduledResource.cpu,<EOL>"<STR_LIT>": container_plan.scheduledResource.ram,<EOL>"<STR_LIT>": container_plan.scheduledResource.disk}<EOL><DEDENT>container = {"<STR_LIT:id>": container_plan.id,<EOL>"<STR_LIT>": instances,<EOL>"<STR_LIT>": required_resource,<EOL>"<STR_LIT>": scheduled_resource}<EOL>containers.append(container)<EOL><DEDENT>packingPlan["<STR_LIT:id>"] = topology.packing_plan.id<EOL>packingPlan["<STR_LIT>"] = containers<EOL>return json.dumps(packingPlan)<EOL>

Returns the representation of packing plan that will
be returned from Tracker.


<EOL>if not topology.execution_state:<EOL><INDENT>Log.info("<STR_LIT>" + topology.name)<EOL>return<EOL><DEDENT>Log.info("<STR_LIT>" + topology.name)<EOL>has_physical_plan = True<EOL>if not topology.physical_plan:<EOL><INDENT>has_physical_plan = False<EOL><DEDENT>Log.info("<STR_LIT>" + topology.name)<EOL>has_packing_plan = True<EOL>if not topology.packing_plan:<EOL><INDENT>has_packing_plan = False<EOL><DEDENT>has_tmaster_location = True<EOL>if not topology.tmaster:<EOL><INDENT>has_tmaster_location = False<EOL><DEDENT>has_scheduler_location = True<EOL>if not topology.scheduler_location:<EOL><INDENT>has_scheduler_location = False<EOL><DEDENT>topologyInfo = {<EOL>"<STR_LIT:name>": topology.name,<EOL>"<STR_LIT:id>": topology.id,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None,<EOL>}<EOL>executionState = self.extract_execution_state(topology)<EOL>executionState["<STR_LIT>"] = has_physical_plan<EOL>executionState["<STR_LIT>"] = has_packing_plan<EOL>executionState["<STR_LIT>"] = has_tmaster_location<EOL>executionState["<STR_LIT>"] = has_scheduler_location<EOL>executionState["<STR_LIT:status>"] = topology.get_status()<EOL>topologyInfo["<STR_LIT>"] = self.extract_metadata(topology)<EOL>topologyInfo["<STR_LIT>"] = self.extract_runtime_state(topology)<EOL>topologyInfo["<STR_LIT>"] = executionState<EOL>topologyInfo["<STR_LIT>"] = self.extract_logical_plan(topology)<EOL>topologyInfo["<STR_LIT>"] = self.extract_physical_plan(topology)<EOL>topologyInfo["<STR_LIT>"] = self.extract_packing_plan(topology)<EOL>topologyInfo["<STR_LIT>"] = self.extract_tmaster(topology)<EOL>topologyInfo["<STR_LIT>"] = self.extract_scheduler_location(topology)<EOL>self.topologyInfos[(topology.name, topology.state_manager_name)] = topologyInfo<EOL>

Extracts info from the stored proto states and
convert it into representation that is exposed using
the API.
This method is called on any change for the topology.
For example, when a container moves and its host or some
port changes. All the information is parsed all over
again and cache is updated.


def getTopologyInfo(self, topologyName, cluster, role, environ):

<EOL>for (topology_name, _), topologyInfo in self.topologyInfos.items():<EOL><INDENT>executionState = topologyInfo["<STR_LIT>"]<EOL>if (topologyName == topology_name and<EOL>cluster == executionState["<STR_LIT>"] and<EOL>environ == executionState["<STR_LIT>"]):<EOL><INDENT>if not role or executionState.get("<STR_LIT>") == role:<EOL><INDENT>return topologyInfo<EOL><DEDENT><DEDENT><DEDENT>if role is not None:<EOL><INDENT>Log.info("<STR_LIT>""<STR_LIT>",<EOL>topologyName, cluster, role, environ)<EOL><DEDENT>else:<EOL><INDENT>Log.info("<STR_LIT>""<STR_LIT>", topologyName, cluster, environ)<EOL><DEDENT>raise Exception("<STR_LIT>")<EOL>

Returns the JSON representation of a topology
by its name, cluster, environ, and an optional role parameter.
Raises exception if no such topology is found.


if not U in self.edges:<EOL><INDENT>self.edges[U] = set()<EOL><DEDENT>if not V in self.edges:<EOL><INDENT>self.edges[V] = set()<EOL><DEDENT>if not V in self.edges[U]:<EOL><INDENT>self.edges[U].add(V)<EOL><DEDENT>


bfs_queue = [[U, <NUM_LIT:0>]]  <EOL>visited = set()<EOL>max_depth = <NUM_LIT:0><EOL>while bfs_queue:<EOL><INDENT>[V, depth] = bfs_queue.pop()<EOL>if max_depth < depth:<EOL><INDENT>max_depth = depth<EOL><DEDENT>visited.add(V)<EOL>adj_set = self.edges[V]<EOL>for W in adj_set:<EOL><INDENT>if W not in visited:<EOL><INDENT>bfs_queue.append([W, depth + <NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT>return max_depth<EOL>

Returns the maximum distance between any vertex and U in the connected
component containing U
:param U:
:return:


diameter = <NUM_LIT:0><EOL>for U in self.edges:<EOL><INDENT>depth = self.bfs_depth(U)<EOL>if depth > diameter:<EOL><INDENT>diameter = depth<EOL><DEDENT><DEDENT>return diameter<EOL>

Returns the maximum distance between any vertex and U in the connected
component containing U
:return:


def __init__(self, componentName, metricName, instance, start, end, timeline):

self.componentName = componentName<EOL>self.metricName = metricName<EOL>self.instance = instance<EOL>self.start = start<EOL>self.end = end<EOL>self.timeline = self.floorTimestamps(start, end, timeline)<EOL>

Takes (componentName, metricname, instance, timeline)


def floorTimestamps(self, start, end, timeline):

ret = {}<EOL>for timestamp, value in timeline.items():<EOL><INDENT>ts = timestamp / <NUM_LIT> * <NUM_LIT><EOL>if start <= ts <= end:<EOL><INDENT>ret[ts] = value<EOL><DEDENT><DEDENT>return ret<EOL>


def setDefault(self, constant, start, end):

starttime = start / <NUM_LIT> * <NUM_LIT><EOL>if starttime < start:<EOL><INDENT>starttime += <NUM_LIT><EOL><DEDENT>endtime = end / <NUM_LIT> * <NUM_LIT><EOL>while starttime <= endtime:<EOL><INDENT>if starttime not in self.timeline or self.timeline[starttime] == <NUM_LIT:0>:<EOL><INDENT>self.timeline[starttime] = constant<EOL><DEDENT>starttime += <NUM_LIT><EOL><DEDENT>


@tornado.gen.coroutine<EOL><INDENT>def execute(self, tracker, tmaster, start, end):<DEDENT>

raise Exception("<STR_LIT>")<EOL>


return True<EOL>

Returns True. This is just usefule for checking that an object is an operator or not.


printable = string.ascii_letters + string.digits + string.punctuation + '<STR_LIT:U+0020>'<EOL>return '<STR_LIT>'.join(ch if ch in printable else r'<STR_LIT>'.format(ord(ch)) for ch in bin_str)<EOL>


<EOL>pplan = topologyInfo["<STR_LIT>"]<EOL>stmgrId = pplan["<STR_LIT>"][instance_id]["<STR_LIT>"]<EOL>host = pplan["<STR_LIT>"][stmgrId]["<STR_LIT:host>"]<EOL>shell_port = pplan["<STR_LIT>"][stmgrId]["<STR_LIT>"]<EOL>return "<STR_LIT>" % (host, shell_port)<EOL>

Makes the http endpoint for the heron shell
if shell port is present, otherwise returns None.


def make_shell_job_url(host, shell_port, _):

if not shell_port:<EOL><INDENT>return None<EOL><DEDENT>return "<STR_LIT>" % (host, shell_port)<EOL>

Make the job url from the info
stored in stmgr. This points to dir from where
all the processes are started.
If shell port is not present, it returns None.


def make_shell_logfiles_url(host, shell_port, _, instance_id=None):

if not shell_port:<EOL><INDENT>return None<EOL><DEDENT>if not instance_id:<EOL><INDENT>return "<STR_LIT>" % (host, shell_port)<EOL><DEDENT>else:<EOL><INDENT>return "<STR_LIT>" % (host, shell_port, instance_id)<EOL><DEDENT>

Make the url for log-files in heron-shell
from the info stored in stmgr.
If no instance_id is provided, the link will
be to the dir for the whole container.
If shell port is not present, it returns None.


def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length):

return "<STR_LIT>" %(host, shell_port, instance_id, offset, length)<EOL>

Make the url for log-file data in heron-shell
from the info stored in stmgr.


def make_shell_filestats_url(host, shell_port, path):

return "<STR_LIT>" % (host, shell_port, path)<EOL>

Make the url for filestats data in heron-shell
from the info stored in stmgr.


def make_viz_dashboard_url(name, cluster, environ):

return "<STR_LIT>"<EOL>

Link to the dashboard. Must override to return a valid url.


return x<EOL>

This will return the input arg
:return: input argument


command = ['<STR_LIT>', '<STR_LIT>', x]<EOL>p = subprocess.Popen(command, stdout=subprocess.PIPE)<EOL>output, _ = p.communicate()<EOL>lines = output.split("<STR_LIT:\n>")<EOL>return lines[<NUM_LIT:0>]<EOL>

This will return the path of input arg for windows
:return: the path in windows


if sys.platform == '<STR_LIT>':<EOL><INDENT>return cygpath(x)<EOL><DEDENT>return identity(x)<EOL>

This will return the class path depending on the platform
:return: the class path


path = "<STR_LIT:/>".join(os.path.realpath(__file__).split('<STR_LIT:/>')[:-<NUM_LIT:8>])<EOL>return normalized_class_path(path)<EOL>

This will extract heron tracker directory from .pex file.
:return: root location for heron-tools.


bin_path = os.path.join(get_heron_tracker_dir(), BIN_DIR)<EOL>return bin_path<EOL>

This will provide heron tracker bin directory from .pex file.
:return: absolute path of heron lib directory


conf_path = os.path.join(get_heron_tracker_dir(), CONF_DIR)<EOL>return conf_path<EOL>

This will provide heron tracker conf directory from .pex file.
:return: absolute path of heron conf directory


expanded_config_file_path = os.path.expanduser(config_file)<EOL>if not os.path.lexists(expanded_config_file_path):<EOL><INDENT>return None<EOL><DEDENT>configs = {}<EOL>with open(expanded_config_file_path, '<STR_LIT:r>') as f:<EOL><INDENT>configs = yaml.load(f)<EOL><DEDENT>return configs<EOL>

This will parse the config file for the tracker
:return: the config or None if the file is not found


parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=True<EOL>)<EOL>cli_args.add_titles(parser)<EOL>parser_action = parser.add_subparsers()<EOL>parser_cluster = parser_action.add_parser(<EOL>Action.CLUSTER,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter,<EOL>)<EOL>parser_cluster.set_defaults(action=Action.CLUSTER)<EOL>parser_set = parser_action.add_parser(<EOL>Action.SET,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_set.set_defaults(action=Action.SET)<EOL>parser_template = parser_action.add_parser(<EOL>Action.TEMPLATE,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_template.set_defaults(action=Action.TEMPLATE)<EOL>parser_cluster.add_argument(<EOL>TYPE,<EOL>type=str,<EOL>choices={Cluster.START, Cluster.STOP},<EOL>help="<EOL>ces supports the following:<EOL>art     - Start standalone Heron cluster<EOL>op      - Stop standalone Heron cluster<EOL>)<EOL>parser_template.add_argument(<EOL>TYPE,<EOL>type=str,<EOL>choices={"<STR_LIT>"},<EOL>)<EOL>parser_get = parser_action.add_parser(<EOL>Action.GET,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_get.set_defaults(action=Action.GET)<EOL>parser_get.add_argument(<EOL>TYPE,<EOL>type=str,<EOL>choices={Get.SERVICE_URL, Get.HERON_TRACKER_URL, Get.HERON_UI_URL},<EOL>help="""<STR_LIT>"""<EOL>)<EOL>parser_info = parser_action.add_parser(<EOL>Action.INFO,<EOL>help='<STR_LIT>',<EOL>add_help=True,<EOL>formatter_class=argparse.RawTextHelpFormatter<EOL>)<EOL>parser_info.set_defaults(action=Action.INFO)<EOL>add_additional_args([parser_set, parser_cluster, parser_template, parser_get, parser_info])<EOL>parser.set_defaults(subcommand='<STR_LIT>')<EOL>return parser<EOL>

Create a subparser for the standalone command
:param subparsers:
:return:


def run(command, parser, cl_args, unknown_args):

action = cl_args["<STR_LIT:action>"]<EOL>if action == Action.SET:<EOL><INDENT>call_editor(get_inventory_file(cl_args))<EOL>update_config_files(cl_args)<EOL><DEDENT>elif action == Action.CLUSTER:<EOL><INDENT>action_type = cl_args["<STR_LIT:type>"]<EOL>if action_type == Cluster.START:<EOL><INDENT>start_cluster(cl_args)<EOL><DEDENT>elif action_type == Cluster.STOP:<EOL><INDENT>if check_sure(cl_args, "<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"):<EOL><INDENT>stop_cluster(cl_args)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % action_type)<EOL><DEDENT><DEDENT>elif action == Action.TEMPLATE:<EOL><INDENT>update_config_files(cl_args)<EOL><DEDENT>elif action == Action.GET:<EOL><INDENT>action_type = cl_args["<STR_LIT:type>"]<EOL>if action_type == Get.SERVICE_URL:<EOL><INDENT>print(get_service_url(cl_args))<EOL><DEDENT>elif action_type == Get.HERON_UI_URL:<EOL><INDENT>print(get_heron_ui_url(cl_args))<EOL><DEDENT>elif action_type == Get.HERON_TRACKER_URL:<EOL><INDENT>print(get_heron_tracker_url(cl_args))<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % action_type)<EOL><DEDENT><DEDENT>elif action == Action.INFO:<EOL><INDENT>print_cluster_info(cl_args)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % action)<EOL><DEDENT>return SimpleResult(Status.Ok)<EOL>


slave_config_template = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>slave_config_actual = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>masters_in_quotes = ['<STR_LIT>' % master for master in masters]<EOL>template_file(slave_config_template, slave_config_actual,<EOL>{"<STR_LIT>": "<STR_LIT:U+002CU+0020>".join(masters_in_quotes)})<EOL>


single_master = masters[<NUM_LIT:0>]<EOL>scheduler_config_actual = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>scheduler_config_template = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>template_file(scheduler_config_template, scheduler_config_actual,<EOL>{"<STR_LIT>": "<STR_LIT>" % single_master})<EOL>


single_master = masters[<NUM_LIT:0>]<EOL>uploader_config_template = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>uploader_config_actual = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>template_file(uploader_config_template, uploader_config_actual,<EOL>{"<STR_LIT>": "<STR_LIT>" % single_master})<EOL>


def template_apiserver_hcl(cl_args, masters, zookeepers):

single_master = masters[<NUM_LIT:0>]<EOL>apiserver_config_template = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>apiserver_config_actual = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>replacements = {<EOL>"<STR_LIT>": '<STR_LIT>' % get_hostname(single_master, cl_args),<EOL>"<STR_LIT>": '<STR_LIT>'<EOL>% config.get_heron_bin_dir()<EOL>if is_self(single_master)<EOL>else '<STR_LIT>'<EOL>% get_remote_home(single_master, cl_args),<EOL>"<STR_LIT>": "<STR_LIT:U+002C>".join(<EOL>['<STR_LIT:%s>' % zk if "<STR_LIT::>" in zk else '<STR_LIT>' % zk for zk in zookeepers]),<EOL>"<STR_LIT>": "<STR_LIT>" % single_master<EOL>}<EOL>template_file(apiserver_config_template, apiserver_config_actual, replacements)<EOL>


statemgr_config_file_template = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>statemgr_config_file_actual = "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>template_file(statemgr_config_file_template, statemgr_config_file_actual,<EOL>{"<STR_LIT>": "<STR_LIT:U+002C>".join(<EOL>['<STR_LIT>' % zk if "<STR_LIT::>" in zk else '<STR_LIT>' % zk for zk in zookeepers])})<EOL>


def template_heron_tools_hcl(cl_args, masters, zookeepers):

heron_tools_hcl_template = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>heron_tools_hcl_actual = "<STR_LIT>"% cl_args["<STR_LIT>"]<EOL>single_master = masters[<NUM_LIT:0>]<EOL>template_file(heron_tools_hcl_template, heron_tools_hcl_actual,<EOL>{<EOL>"<STR_LIT>": "<STR_LIT:U+002C>".join(<EOL>['<STR_LIT:%s>' % zk if "<STR_LIT::>" in zk else '<STR_LIT>' % zk for zk in zookeepers]),<EOL>"<STR_LIT>": '<STR_LIT>' % config.get_heron_bin_dir(),<EOL>"<STR_LIT>": '<STR_LIT>' % get_hostname(single_master, cl_args),<EOL>"<STR_LIT>": '<STR_LIT>' % config.get_heron_bin_dir()<EOL>})<EOL>


roles = read_and_parse_roles(cl_args)<EOL>return "<STR_LIT>" % list(roles[Role.MASTERS])[<NUM_LIT:0>]<EOL>


parsed_roles = read_and_parse_roles(cl_args)<EOL>masters = list(parsed_roles[Role.MASTERS])<EOL>slaves = list(parsed_roles[Role.SLAVES])<EOL>zookeepers = list(parsed_roles[Role.ZOOKEEPERS])<EOL>cluster = list(parsed_roles[Role.CLUSTER])<EOL>info = OrderedDict()<EOL>info['<STR_LIT>'] = len(cluster)<EOL>info['<STR_LIT>'] = cluster<EOL>roles = OrderedDict()<EOL>roles['<STR_LIT>'] = masters<EOL>roles['<STR_LIT>'] = slaves<EOL>roles['<STR_LIT>'] = zookeepers<EOL>urls = OrderedDict()<EOL>urls['<STR_LIT>'] = get_service_url(cl_args)<EOL>urls['<STR_LIT>'] = get_heron_ui_url(cl_args)<EOL>urls['<STR_LIT>'] = get_heron_tracker_url(cl_args)<EOL>info['<STR_LIT>'] = roles<EOL>info['<STR_LIT>'] = urls<EOL>print(json.dumps(info, indent=<NUM_LIT:2>))<EOL>


Log.info("<STR_LIT>")<EOL>roles = read_and_parse_roles(cl_args)<EOL>masters = roles[Role.MASTERS]<EOL>slaves = roles[Role.SLAVES]<EOL>dist_nodes = masters.union(slaves)<EOL>if masters:<EOL><INDENT>try:<EOL><INDENT>single_master = list(masters)[<NUM_LIT:0>]<EOL>jobs = get_jobs(cl_args, single_master)<EOL>for job in jobs:<EOL><INDENT>job_id = job["<STR_LIT>"]<EOL>Log.info("<STR_LIT>" % job_id)<EOL>delete_job(cl_args, job_id, single_master)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL>Log.debug(sys.exc_info()[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>for node in dist_nodes:<EOL><INDENT>Log.info("<STR_LIT>" % node)<EOL>if not is_self(node):<EOL><INDENT>cmd = "<STR_LIT>""<STR_LIT>"<EOL>cmd = ssh_remote_execute(cmd, node, cl_args)<EOL><DEDENT>else:<EOL><INDENT>cmd = "<STR_LIT>""<STR_LIT>"<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>Log.info("<STR_LIT>" % node)<EOL>cmd = "<STR_LIT>"<EOL>if not is_self(node):<EOL><INDENT>cmd = ssh_remote_execute(cmd, node, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL><DEDENT>


roles = read_and_parse_roles(cl_args)<EOL>masters = roles[Role.MASTERS]<EOL>slaves = roles[Role.SLAVES]<EOL>zookeepers = roles[Role.ZOOKEEPERS]<EOL>Log.info("<STR_LIT>")<EOL>Log.info("<STR_LIT>" % list(masters))<EOL>Log.info("<STR_LIT>" % list(slaves))<EOL>Log.info("<STR_LIT>" % list(zookeepers))<EOL>if not masters:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>if not slaves:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>if not zookeepers:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>update_config_files(cl_args)<EOL>dist_nodes = list(masters.union(slaves))<EOL>if not (len(dist_nodes) == <NUM_LIT:1> and is_self(dist_nodes[<NUM_LIT:0>])):<EOL><INDENT>distribute_package(roles, cl_args)<EOL><DEDENT>start_master_nodes(masters, cl_args)<EOL>start_slave_nodes(slaves, cl_args)<EOL>start_api_server(masters, cl_args)<EOL>start_heron_tools(masters, cl_args)<EOL>Log.info("<STR_LIT>")<EOL>


<EOL>single_master = list(masters)[<NUM_LIT:0>]<EOL>wait_for_master_to_start(single_master)<EOL>cmd = "<STR_LIT>"% (get_nomad_path(cl_args), get_apiserver_job_file(cl_args))<EOL>Log.info("<STR_LIT>" % single_master)<EOL>if not is_self(single_master):<EOL><INDENT>cmd = ssh_remote_execute(cmd, single_master, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (single_master, output[<NUM_LIT:1>]))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>wait_for_job_to_start(single_master, "<STR_LIT>")<EOL>Log.info("<STR_LIT>")<EOL>


single_master = list(masters)[<NUM_LIT:0>]<EOL>wait_for_master_to_start(single_master)<EOL>cmd = "<STR_LIT>"% (get_nomad_path(cl_args), get_heron_tools_job_file(cl_args))<EOL>Log.info("<STR_LIT>" % single_master)<EOL>if not is_self(single_master):<EOL><INDENT>cmd = ssh_remote_execute(cmd, single_master, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (single_master, output[<NUM_LIT:1>]))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>wait_for_job_to_start(single_master, "<STR_LIT>")<EOL>Log.info("<STR_LIT>")<EOL>


Log.info("<STR_LIT>")<EOL>masters = roles[Role.MASTERS]<EOL>slaves = roles[Role.SLAVES]<EOL>tar_file = tempfile.NamedTemporaryFile(suffix="<STR_LIT>").name<EOL>Log.debug("<STR_LIT>" % (cl_args["<STR_LIT>"], tar_file))<EOL>make_tarfile(tar_file, cl_args["<STR_LIT>"])<EOL>dist_nodes = masters.union(slaves)<EOL>scp_package(tar_file, dist_nodes, cl_args)<EOL>


i = <NUM_LIT:0><EOL>while True:<EOL><INDENT>try:<EOL><INDENT>r = requests.get("<STR_LIT>" % single_master)<EOL>if r.status_code == <NUM_LIT:200>:<EOL><INDENT>break<EOL><DEDENT><DEDENT>except:<EOL><INDENT>Log.debug(sys.exc_info()[<NUM_LIT:0>])<EOL>Log.info("<STR_LIT>" % i)<EOL>time.sleep(<NUM_LIT:1>)<EOL>if i > <NUM_LIT:10>:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT><DEDENT>i = i + <NUM_LIT:1><EOL><DEDENT>


i = <NUM_LIT:0><EOL>while True:<EOL><INDENT>try:<EOL><INDENT>r = requests.get("<STR_LIT>" % (single_master, job))<EOL>if r.status_code == <NUM_LIT:200> and r.json()["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError()<EOL><DEDENT><DEDENT>except:<EOL><INDENT>Log.debug(sys.exc_info()[<NUM_LIT:0>])<EOL>Log.info("<STR_LIT>" % (job, i))<EOL>time.sleep(<NUM_LIT:1>)<EOL>if i > <NUM_LIT:20>:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT><DEDENT>i = i + <NUM_LIT:1><EOL><DEDENT>


def scp_package(package_file, destinations, cl_args):

pids = []<EOL>for dest in destinations:<EOL><INDENT>if is_self(dest):<EOL><INDENT>continue<EOL><DEDENT>Log.info("<STR_LIT>" % dest)<EOL>file_path = "<STR_LIT>"<EOL>dest_file_path = "<STR_LIT>" % (dest, file_path)<EOL>remote_cmd = "<STR_LIT>""<STR_LIT>" % (file_path)<EOL>cmd = '<STR_LIT>'% (scp_cmd(package_file, dest_file_path, cl_args),<EOL>ssh_remote_execute(remote_cmd, dest, cl_args))<EOL>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>pids.append({"<STR_LIT>": pid, "<STR_LIT>": dest})<EOL><DEDENT>errors = []<EOL>for entry in pids:<EOL><INDENT>pid = entry["<STR_LIT>"]<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>errors.append("<STR_LIT>" % (entry["<STR_LIT>"], output[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>if errors:<EOL><INDENT>for error in errors:<EOL><INDENT>Log.error(error)<EOL><DEDENT>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>


with tarfile.open(output_filename, "<STR_LIT>") as tar:<EOL><INDENT>tar.add(source_dir, arcname=os.path.basename(source_dir))<EOL><DEDENT>


pids = []<EOL>for master in masters:<EOL><INDENT>Log.info("<STR_LIT>" % master)<EOL>cmd = "<STR_LIT>"% (get_nomad_path(cl_args), get_nomad_master_config_file(cl_args))<EOL>if not is_self(master):<EOL><INDENT>cmd = ssh_remote_execute(cmd, master, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>pids.append({"<STR_LIT>": pid, "<STR_LIT>": master})<EOL><DEDENT>errors = []<EOL>for entry in pids:<EOL><INDENT>pid = entry["<STR_LIT>"]<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>errors.append("<STR_LIT>" % (entry["<STR_LIT>"], output[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>if errors:<EOL><INDENT>for error in errors:<EOL><INDENT>Log.error(error)<EOL><DEDENT>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>


pids = []<EOL>for slave in slaves:<EOL><INDENT>Log.info("<STR_LIT>" % slave)<EOL>cmd = "<STR_LIT>"% (get_nomad_path(cl_args), get_nomad_slave_config_file(cl_args))<EOL>if not is_self(slave):<EOL><INDENT>cmd = ssh_remote_execute(cmd, slave, cl_args)<EOL><DEDENT>Log.debug(cmd)<EOL>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>pids.append({"<STR_LIT>": pid, "<STR_LIT>": slave})<EOL><DEDENT>errors = []<EOL>for entry in pids:<EOL><INDENT>pid = entry["<STR_LIT>"]<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>Log.debug("<STR_LIT>" % (return_code, output))<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>errors.append("<STR_LIT>" % (entry["<STR_LIT>"], output[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>if errors:<EOL><INDENT>for error in errors:<EOL><INDENT>Log.error(error)<EOL><DEDENT>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>


roles = dict()<EOL>with open(get_inventory_file(cl_args), '<STR_LIT:r>') as stream:<EOL><INDENT>try:<EOL><INDENT>roles = yaml.load(stream)<EOL><DEDENT>except yaml.YAMLError as exc:<EOL><INDENT>Log.error("<STR_LIT>" % exc)<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if Role.ZOOKEEPERS not in roles or not roles[Role.ZOOKEEPERS]:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>if Role.CLUSTER not in roles or not roles[Role.CLUSTER]:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>roles[Role.MASTERS] = set([roles[Role.CLUSTER][<NUM_LIT:0>]])<EOL>roles[Role.SLAVES] = set(roles[Role.CLUSTER])<EOL>roles[Role.ZOOKEEPERS] = set(roles[Role.ZOOKEEPERS])<EOL>roles[Role.CLUSTER] = set(roles[Role.CLUSTER])<EOL>return roles<EOL>


lines = []<EOL>with open(file_path, "<STR_LIT:r>") as tf:<EOL><INDENT>lines = [line.strip("<STR_LIT:\n>") for line in tf.readlines() if not line.startswith("<STR_LIT:#>")]<EOL>lines = [line for line in lines if line]<EOL><DEDENT>return lines<EOL>


EDITOR = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL>with open(file_path, '<STR_LIT>') as tf:<EOL><INDENT>call([EDITOR, tf.name])<EOL><DEDENT>


return "<STR_LIT>" % cl_args["<STR_LIT>"]<EOL>


def ssh_remote_execute(cmd, host, cl_args):

ssh = '<STR_LIT>' % (host, cmd)<EOL>return ssh<EOL>


scp = '<STR_LIT>' % (src, dest)<EOL>return scp<EOL>


return "<STR_LIT>" % config.get_heron_bin_dir()<EOL>


return "<STR_LIT>" % config.get_heron_conf_dir()<EOL>


cmd = "<STR_LIT>"<EOL>if not is_self(host):<EOL><INDENT>cmd = ssh_remote_execute(cmd, host, cl_args)<EOL><DEDENT>pid = subprocess.Popen(cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (host, output))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>return output[<NUM_LIT:0>].strip("<STR_LIT:\n>")<EOL>


return socket.gethostbyname(socket.gethostname())<EOL>


return socket.gethostname()<EOL>


if is_self(ip_addr):<EOL><INDENT>return get_self_hostname()<EOL><DEDENT>cmd = "<STR_LIT>"<EOL>ssh_cmd = ssh_remote_execute(cmd, ip_addr, cl_args)<EOL>pid = subprocess.Popen(ssh_cmd,<EOL>shell=True,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>return_code = pid.wait()<EOL>output = pid.communicate()<EOL>if return_code != <NUM_LIT:0>:<EOL><INDENT>Log.error("<STR_LIT>" % (ip_addr, output))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT>return output[<NUM_LIT:0>].strip("<STR_LIT:\n>")<EOL>


ips = []<EOL>for i in netifaces.interfaces():<EOL><INDENT>entry = netifaces.ifaddresses(i)<EOL>if netifaces.AF_INET in entry:<EOL><INDENT>for ipv4 in entry[netifaces.AF_INET]:<EOL><INDENT>if "<STR_LIT>" in ipv4:<EOL><INDENT>ips.append(ipv4["<STR_LIT>"])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return addr in ips or addr == get_self_hostname()<EOL>


return {<EOL>'<STR_LIT>': standalone,<EOL>}<EOL>

Create a map of command names and handlers


parser = argparse.ArgumentParser(<EOL>prog='<STR_LIT>',<EOL>epilog=HELP_EPILOG,<EOL>formatter_class=config.SubcommandHelpFormatter,<EOL>add_help=True)<EOL>subparsers = parser.add_subparsers(<EOL>title="<STR_LIT>",<EOL>metavar='<STR_LIT>')<EOL>command_list = sorted(command_handlers.items())<EOL>for command in command_list:<EOL><INDENT>command[<NUM_LIT:1>].create_parser(subparsers)<EOL><DEDENT>return parser<EOL>


def run(handlers, command, parser, command_args, unknown_args):

if command in handlers:<EOL><INDENT>return handlers[command].run(command, parser, command_args, unknown_args)<EOL><DEDENT>else:<EOL><INDENT>err_context = '<STR_LIT>' % command<EOL>return result.SimpleResult(result.Status.InvocationError, err_context)<EOL><DEDENT>

Run the command
:param command:
:param parser:
:param command_args:
:param unknown_args:
:return:


for cur_file in files:<EOL><INDENT>if os.path.isdir(cur_file):<EOL><INDENT>shutil.rmtree(cur_file)<EOL><DEDENT>else:<EOL><INDENT>shutil.rmtree(os.path.dirname(cur_file))<EOL><DEDENT><DEDENT>


if not config.check_java_home_set():<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if not config.check_release_file_exists():<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

Check whether the environment variables are set
:return:


<EOL>check_environment()<EOL>parser = create_parser(handlers)<EOL>if len(sys.argv[<NUM_LIT:1>:]) == <NUM_LIT:0>:<EOL><INDENT>parser.print_help()<EOL>return <NUM_LIT:0><EOL><DEDENT>sys.argv = config.insert_bool_values(sys.argv)<EOL>try:<EOL><INDENT>args, unknown_args = parser.parse_known_args()<EOL><DEDENT>except ValueError as ex:<EOL><INDENT>Log.error("<STR_LIT>", str(ex))<EOL>Log.debug(traceback.format_exc())<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>command_line_args = vars(args)<EOL>log.set_logging_level(command_line_args)<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>command = command_line_args['<STR_LIT>']<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>results = run(handlers, command, parser, command_line_args, unknown_args)<EOL>return <NUM_LIT:0> if result.is_successful(results) else <NUM_LIT:1><EOL>


configcommand = cl_args.get('<STR_LIT>', None)<EOL>if configcommand == '<STR_LIT>':<EOL><INDENT>return _set(cl_args)<EOL><DEDENT>elif configcommand == '<STR_LIT>':<EOL><INDENT>return _unset(cl_args)<EOL><DEDENT>else:<EOL><INDENT>return _list(cl_args)<EOL><DEDENT>

:param command:
:param parser:
:param args:
:param unknown_args:
:return:


client_confs = dict()<EOL>client_confs[cluster] = cliconfig.cluster_config(cluster)<EOL>if cl_args.get('<STR_LIT>', None):<EOL><INDENT>client_confs[cluster]['<STR_LIT>'] = cl_args['<STR_LIT>']<EOL><DEDENT>return client_confs<EOL>

Read the cluster definition for server mode
:param cluster:
:param cl_args:
:param config_file:
:return:


config_path = config.get_heron_cluster_conf_dir(cluster, config_path)<EOL>if not os.path.isdir(config_path):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Check the cluster definition for direct mode
:param cluster:
:param config_path:
:return:


<EOL>parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>return parser<EOL>


cluster = cl_args['<STR_LIT>']<EOL>if cluster:<EOL><INDENT>config_file = config.heron_rc_file()<EOL>client_confs = dict()<EOL>client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args, config_file)<EOL>if not client_confs[cluster]:<EOL><INDENT>Log.error('<STR_LIT>', cluster, config_file)<EOL>return SimpleResult(Status.HeronError)<EOL><DEDENT>if not '<STR_LIT>' in client_confs[cluster]:<EOL><INDENT>Log.error('<STR_LIT>', cluster, config_file)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>service_endpoint = cl_args['<STR_LIT>']<EOL>service_apiurl = service_endpoint + rest.ROUTE_SIGNATURES[command][<NUM_LIT:1>]<EOL>service_method = rest.ROUTE_SIGNATURES[command][<NUM_LIT:0>]<EOL>try:<EOL><INDENT>r = service_method(service_apiurl)<EOL>if r.status_code != requests.codes.ok:<EOL><INDENT>Log.error(r.json().get('<STR_LIT:message>', "<STR_LIT>" % r.status_code))<EOL><DEDENT>sorted_items = sorted(r.json().items(), key=lambda tup: tup[<NUM_LIT:0>])<EOL>for key, value in sorted_items:<EOL><INDENT>print("<STR_LIT>" % (key, value))<EOL><DEDENT><DEDENT>except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:<EOL><INDENT>Log.error(err)<EOL>return SimpleResult(Status.HeronError)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>config.print_build_info()<EOL><DEDENT>return SimpleResult(Status.Ok)<EOL>


if cl_args['<STR_LIT>']:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>return "<STR_LIT>"<EOL>

Depending on the mode of launching a topology provide a message
:param cl_args:
:return:


Create a subparser for the submit command
:param subparsers:
:return:


def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name):

<EOL>topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, '<STR_LIT>'))<EOL>release_yaml_file = cl_args['<STR_LIT>']<EOL>config_path = cl_args['<STR_LIT>']<EOL>tar_pkg_files = [topology_file, topology_defn_file]<EOL>generated_config_files = [release_yaml_file, cl_args['<STR_LIT>']]<EOL>config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)<EOL>args = [<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", config.get_heron_dir(),<EOL>"<STR_LIT>", config_path,<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", release_yaml_file,<EOL>"<STR_LIT>", topology_pkg_path,<EOL>"<STR_LIT>", topology_defn_file,<EOL>"<STR_LIT>", os.path.basename(topology_file)   <EOL>]<EOL>if Log.getEffectiveLevel() == logging.DEBUG:<EOL><INDENT>args.append("<STR_LIT>")<EOL><DEDENT>if cl_args["<STR_LIT>"]:<EOL><INDENT>args.append("<STR_LIT>")<EOL>if "<STR_LIT>" in cl_args:<EOL><INDENT>args += ["<STR_LIT>", cl_args["<STR_LIT>"]]<EOL><DEDENT><DEDENT>lib_jars = config.get_heron_libs(<EOL>jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()<EOL>)<EOL>extra_jars = cl_args['<STR_LIT>'].split('<STR_LIT::>')<EOL>main_class = '<STR_LIT>'<EOL>res = execute.heron_class(<EOL>class_name=main_class,<EOL>lib_jars=lib_jars,<EOL>extra_jars=extra_jars,<EOL>args=args,<EOL>java_defines=[])<EOL>err_ctxt = "<STR_LIT>" % (topology_name, launch_mode_msg(cl_args))<EOL>succ_ctxt = "<STR_LIT>" % (topology_name, launch_mode_msg(cl_args))<EOL>res.add_context(err_ctxt, succ_ctxt)<EOL>return res<EOL>

Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param tmp_dir:
:param topology_file:
:param topology_defn_file:
:param topology_name:
:return:


def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name):

service_apiurl = cl_args['<STR_LIT>'] + rest.ROUTE_SIGNATURES['<STR_LIT>'][<NUM_LIT:1>]<EOL>service_method = rest.ROUTE_SIGNATURES['<STR_LIT>'][<NUM_LIT:0>]<EOL>data = dict(<EOL>name=topology_name,<EOL>cluster=cl_args['<STR_LIT>'],<EOL>role=cl_args['<STR_LIT>'],<EOL>environment=cl_args['<STR_LIT>'],<EOL>user=cl_args['<STR_LIT>'],<EOL>)<EOL>Log.info("<STR_LIT>" + str(cl_args))<EOL>overrides = dict()<EOL>if '<STR_LIT>' in cl_args:<EOL><INDENT>overrides = config.parse_override_config(cl_args['<STR_LIT>'])<EOL><DEDENT>if overrides:<EOL><INDENT>data.update(overrides)<EOL><DEDENT>if cl_args['<STR_LIT>']:<EOL><INDENT>data["<STR_LIT>"] = True<EOL><DEDENT>files = dict(<EOL>definition=open(topology_defn_file, '<STR_LIT:rb>'),<EOL>topology=open(topology_file, '<STR_LIT:rb>'),<EOL>)<EOL>err_ctxt = "<STR_LIT>" % (topology_name, launch_mode_msg(cl_args))<EOL>succ_ctxt = "<STR_LIT>" % (topology_name, launch_mode_msg(cl_args))<EOL>try:<EOL><INDENT>r = service_method(service_apiurl, data=data, files=files)<EOL>ok = r.status_code is requests.codes.ok<EOL>created = r.status_code is requests.codes.created<EOL>s = Status.Ok if created or ok else Status.HeronError<EOL>if s is Status.HeronError:<EOL><INDENT>Log.error(r.json().get('<STR_LIT:message>', "<STR_LIT>" % r.status_code))<EOL><DEDENT>elif ok:<EOL><INDENT>print(r.json().get("<STR_LIT>"))<EOL><DEDENT><DEDENT>except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:<EOL><INDENT>Log.error(err)<EOL>return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)<EOL><DEDENT>return SimpleResult(s, err_ctxt, succ_ctxt)<EOL>

Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param topology_file:
:param topology_defn_file:
:param topology_name:
:return:


def launch_topologies(cl_args, topology_file, tmp_dir):

<EOL>defn_files = glob.glob(tmp_dir + '<STR_LIT>')<EOL>if len(defn_files) == <NUM_LIT:0>:<EOL><INDENT>return SimpleResult(Status.HeronError, "<STR_LIT>" % tmp_dir)<EOL><DEDENT>results = []<EOL>for defn_file in defn_files:<EOL><INDENT>topology_defn = topology_pb2.Topology()<EOL>try:<EOL><INDENT>handle = open(defn_file, "<STR_LIT:rb>")<EOL>topology_defn.ParseFromString(handle.read())<EOL>handle.close()<EOL><DEDENT>except Exception as e:<EOL><INDENT>err_context = "<STR_LIT>" % (defn_file, e)<EOL>return SimpleResult(Status.HeronError, err_context)<EOL><DEDENT>Log.info("<STR_LIT>", topology_defn.name, launch_mode_msg(cl_args))<EOL>if cl_args['<STR_LIT>'] == config.SERVER_MODE:<EOL><INDENT>res = launch_topology_server(<EOL>cl_args, topology_file, defn_file, topology_defn.name)<EOL><DEDENT>else:<EOL><INDENT>res = launch_a_topology(<EOL>cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)<EOL><DEDENT>results.append(res)<EOL><DEDENT>return results<EOL>

Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)


def submit_fatjar(cl_args, unknown_args, tmp_dir):

<EOL>topology_file = cl_args['<STR_LIT>']<EOL>main_class = cl_args['<STR_LIT>']<EOL>res = execute.heron_class(<EOL>class_name=main_class,<EOL>lib_jars=config.get_heron_libs(jars.topology_jars()),<EOL>extra_jars=[topology_file],<EOL>args=tuple(unknown_args),<EOL>java_defines=cl_args['<STR_LIT>'])<EOL>result.render(res)<EOL>if not result.is_successful(res):<EOL><INDENT>err_context = ("<STR_LIT>""<STR_LIT>") % (main_class, topology_file)<EOL>res.add_context(err_context)<EOL>return res<EOL><DEDENT>results = launch_topologies(cl_args, topology_file, tmp_dir)<EOL>return results<EOL>

We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.

This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:


def submit_tar(cl_args, unknown_args, tmp_dir):

<EOL>topology_file = cl_args['<STR_LIT>']<EOL>java_defines = cl_args['<STR_LIT>']<EOL>main_class = cl_args['<STR_LIT>']<EOL>res = execute.heron_tar(<EOL>main_class,<EOL>topology_file,<EOL>tuple(unknown_args),<EOL>tmp_dir,<EOL>java_defines)<EOL>result.render(res)<EOL>if not result.is_successful(res):<EOL><INDENT>err_context = ("<STR_LIT>""<STR_LIT>") % (main_class, topology_file)<EOL>res.add_context(err_context)<EOL>return res<EOL><DEDENT>return launch_topologies(cl_args, topology_file, tmp_dir)<EOL>

Extract and execute the java files inside the tar and then add topology
definition file created by running submitTopology

We use the packer to make a package for the tar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
This will run the jar file with the topology class name.

The submitter inside will write out the topology defn file to a location
that we specify. Then we write the topology defn file to a well known
packer location. We then write to appropriate places in zookeeper
and launch the aurora jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:


Log.debug("<STR_LIT>", cl_args)<EOL>topology_file = cl_args['<STR_LIT>']<EOL>if urlparse.urlparse(topology_file).scheme:<EOL><INDENT>cl_args['<STR_LIT>'] = download(topology_file, cl_args['<STR_LIT>'])<EOL>topology_file = cl_args['<STR_LIT>']<EOL>Log.debug("<STR_LIT>", topology_file)<EOL><DEDENT>if not os.path.isfile(topology_file):<EOL><INDENT>err_context = "<STR_LIT>" % topology_file<EOL>return SimpleResult(Status.InvocationError, err_context)<EOL><DEDENT>jar_type = topology_file.endswith("<STR_LIT>")<EOL>tar_type = topology_file.endswith("<STR_LIT>") or topology_file.endswith("<STR_LIT>")<EOL>pex_type = topology_file.endswith("<STR_LIT>")<EOL>cpp_type = topology_file.endswith("<STR_LIT>") or topology_file.endswith("<STR_LIT>")<EOL>if not (jar_type or tar_type or pex_type or cpp_type):<EOL><INDENT>_, ext_name = os.path.splitext(topology_file)<EOL>err_context = "<STR_LIT>""<STR_LIT>"% ext_name<EOL>return SimpleResult(Status.InvocationError, err_context)<EOL><DEDENT>if cl_args['<STR_LIT>']:<EOL><INDENT>valid_classpath = classpath.valid_java_classpath(cl_args['<STR_LIT>'])<EOL>if not valid_classpath:<EOL><INDENT>err_context = "<STR_LIT>" %cl_args['<STR_LIT>']<EOL>return SimpleResult(Status.InvocationError, err_context)<EOL><DEDENT><DEDENT>tmp_dir = tempfile.mkdtemp()<EOL>opts.cleaned_up_files.append(tmp_dir)<EOL>if cl_args['<STR_LIT>']:<EOL><INDENT>initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)<EOL><DEDENT>else:<EOL><INDENT>initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)<EOL><DEDENT>opts.set_config('<STR_LIT>', tmp_dir)<EOL>opts.set_config('<STR_LIT>', initial_state)<EOL>opts.set_config('<STR_LIT>', cl_args['<STR_LIT>'])<EOL>opts.set_config('<STR_LIT>', cl_args['<STR_LIT>'])<EOL>if not cl_args['<STR_LIT>']:<EOL><INDENT>cl_args['<STR_LIT>'] = config.get_heron_release_file()<EOL><DEDENT>if jar_type:<EOL><INDENT>return submit_fatjar(cl_args, unknown_args, tmp_dir)<EOL><DEDENT>elif tar_type:<EOL><INDENT>return submit_tar(cl_args, unknown_args, tmp_dir)<EOL><DEDENT>elif cpp_type:<EOL><INDENT>return submit_cpp(cl_args, unknown_args, tmp_dir)<EOL><DEDENT>else:<EOL><INDENT>return submit_pex(cl_args, unknown_args, tmp_dir)<EOL><DEDENT>

Submits the topology to the scheduler
  * Depending on the topology file name extension, we treat the file as a
    fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
  * We upload the topology file to the packer, update zookeeper and launch
    scheduler jobs representing that topology
  * You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:


parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>usage="<STR_LIT>"<EOL>+ "<STR_LIT>"<EOL>+ "<STR_LIT>"<EOL>+ "<STR_LIT>",<EOL>add_help=True)<EOL>args.add_titles(parser)<EOL>args.add_cluster_role_env(parser)<EOL>args.add_topology(parser)<EOL>args.add_config(parser)<EOL>args.add_dry_run(parser)<EOL>args.add_service_url(parser)<EOL>args.add_verbose(parser)<EOL>def parallelism_type(value):<EOL><INDENT>pattern = re.compile(r"<STR_LIT>")<EOL>if not pattern.match(value):<EOL><INDENT>raise argparse.ArgumentTypeError(<EOL>"<STR_LIT>" % value)<EOL><DEDENT>return value<EOL><DEDENT>parser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT>',<EOL>type=parallelism_type,<EOL>required=False,<EOL>help='<STR_LIT>'<EOL>+ '<STR_LIT>')<EOL>def runtime_config_type(value):<EOL><INDENT>pattern = re.compile(r"<STR_LIT>")<EOL>if not pattern.match(value):<EOL><INDENT>raise argparse.ArgumentTypeError(<EOL>"<STR_LIT>"<EOL>% value)<EOL><DEDENT>return value<EOL><DEDENT>parser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT>',<EOL>type=runtime_config_type,<EOL>required=False,<EOL>help='<STR_LIT>'<EOL>+ '<STR_LIT>')<EOL>def container_number_type(value):<EOL><INDENT>pattern = re.compile(r"<STR_LIT>")<EOL>if not pattern.match(value):<EOL><INDENT>raise argparse.ArgumentTypeError(<EOL>"<STR_LIT>"<EOL>% value)<EOL><DEDENT>return value<EOL><DEDENT>parser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT>',<EOL>type=container_number_type,<EOL>required=False,<EOL>help='<STR_LIT>')<EOL>parser.set_defaults(subcommand='<STR_LIT>')<EOL>return parser<EOL>


<EOL>component_parallelism = cl_args['<STR_LIT>']<EOL>runtime_configs = cl_args['<STR_LIT>']<EOL>container_number = cl_args['<STR_LIT>']<EOL>if (component_parallelism and runtime_configs) or (container_number and runtime_configs):<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>")<EOL><DEDENT>dict_extra_args = {}<EOL>nothing_set = True<EOL>if component_parallelism:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': component_parallelism})<EOL>nothing_set = False<EOL><DEDENT>if container_number:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': container_number})<EOL>nothing_set = False<EOL><DEDENT>if runtime_configs:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': runtime_configs})<EOL>nothing_set = False<EOL><DEDENT>if nothing_set:<EOL><INDENT>raise Exception(<EOL>"<STR_LIT>")<EOL><DEDENT>if cl_args['<STR_LIT>']:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': True})<EOL>if '<STR_LIT>' in cl_args:<EOL><INDENT>dict_extra_args.update({'<STR_LIT>': cl_args["<STR_LIT>"]})<EOL><DEDENT><DEDENT>return dict_extra_args<EOL>


list_extra_args = []<EOL>if '<STR_LIT>' in dict_extra_args:<EOL><INDENT>list_extra_args += ["<STR_LIT>",<EOL>'<STR_LIT:U+002C>'.join(dict_extra_args['<STR_LIT>'])]<EOL><DEDENT>if '<STR_LIT>' in dict_extra_args:<EOL><INDENT>list_extra_args += ["<STR_LIT>",<EOL>'<STR_LIT:U+002C>'.join(dict_extra_args['<STR_LIT>'])]<EOL><DEDENT>if '<STR_LIT>' in dict_extra_args:<EOL><INDENT>list_extra_args += ["<STR_LIT>",<EOL>'<STR_LIT:U+002C>'.join(dict_extra_args['<STR_LIT>'])]<EOL><DEDENT>if '<STR_LIT>' in dict_extra_args and dict_extra_args['<STR_LIT>']:<EOL><INDENT>list_extra_args += ['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in dict_extra_args:<EOL><INDENT>list_extra_args += ['<STR_LIT>', dict_extra_args['<STR_LIT>']]<EOL><DEDENT>return list_extra_args<EOL>


Log.debug("<STR_LIT>", cl_args)<EOL>extra_lib_jars = jars.packing_jars()<EOL>action = "<STR_LIT>" % ('<STR_LIT>' if cl_args["<STR_LIT>"] else '<STR_LIT>')<EOL>dict_extra_args = {}<EOL>try:<EOL><INDENT>dict_extra_args = build_extra_args_dict(cl_args)<EOL><DEDENT>except Exception as err:<EOL><INDENT>return SimpleResult(Status.InvocationError, err.message)<EOL><DEDENT>if cl_args['<STR_LIT>'] == config.SERVER_MODE:<EOL><INDENT>return cli_helper.run_server(command, cl_args, action, dict_extra_args)<EOL><DEDENT>else:<EOL><INDENT>list_extra_args = convert_args_dict_to_list(dict_extra_args)<EOL>return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars)<EOL><DEDENT>


opt_list = []<EOL>for (key, value) in config_opts.items():<EOL><INDENT>opt_list.append('<STR_LIT>' % (key, value))<EOL><DEDENT>all_opts = ('<STR_LIT:U+002C>'.join(opt_list)).replace('<STR_LIT:U+0020>', '<STR_LIT>')<EOL>return all_opts<EOL>

Get config opts from the global variable
:return:


global config_opts<EOL>if k in config_opts:<EOL><INDENT>return config_opts[k]<EOL><DEDENT>return None<EOL>

Get config opts from the config map
:param k:
:return:


global config_opts<EOL>config_opts[k] = v<EOL>

Store a config opt in the config map
:param k:
:param v:
:return:


global config_opts<EOL>config_opts = dict()<EOL>

Clear all the config in the config map
:return:


def create_parser(subparsers, action, help_arg):

:param subparsers:
:param action:
:param help_arg:
:return:


def run_server(command, cl_args, action, extra_args=dict()):

topology_name = cl_args['<STR_LIT>']<EOL>service_endpoint = cl_args['<STR_LIT>']<EOL>apiroute = rest.ROUTE_SIGNATURES[command][<NUM_LIT:1>] % (<EOL>cl_args['<STR_LIT>'],<EOL>cl_args['<STR_LIT>'],<EOL>cl_args['<STR_LIT>'],<EOL>topology_name<EOL>)<EOL>service_apiurl = service_endpoint + apiroute<EOL>service_method = rest.ROUTE_SIGNATURES[command][<NUM_LIT:0>]<EOL>data = flatten_args(extra_args)<EOL>err_msg = "<STR_LIT>" % (action, topology_name)<EOL>succ_msg = "<STR_LIT>" % (action, topology_name)<EOL>try:<EOL><INDENT>r = service_method(service_apiurl, data=data)<EOL>s = Status.Ok if r.status_code == requests.codes.ok else Status.HeronError<EOL>if r.status_code != requests.codes.ok:<EOL><INDENT>Log.error(r.json().get('<STR_LIT:message>', "<STR_LIT>" % r.status_code))<EOL><DEDENT><DEDENT>except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:<EOL><INDENT>Log.error(err)<EOL>return SimpleResult(Status.HeronError, err_msg, succ_msg)<EOL><DEDENT>return SimpleResult(s, err_msg, succ_msg)<EOL>

helper function to take action on topologies using REST API
:param command:
:param cl_args:
:param action:        description of action taken
:return:


def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):

topology_name = cl_args['<STR_LIT>']<EOL>new_args = [<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", config.get_heron_dir(),<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", cl_args['<STR_LIT>'],<EOL>"<STR_LIT>", config.get_heron_release_file(),<EOL>"<STR_LIT>", topology_name,<EOL>"<STR_LIT>", command,<EOL>]<EOL>new_args += extra_args<EOL>lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())<EOL>lib_jars += extra_lib_jars<EOL>if Log.getEffectiveLevel() == logging.DEBUG:<EOL><INDENT>new_args.append("<STR_LIT>")<EOL><DEDENT>result = execute.heron_class(<EOL>'<STR_LIT>',<EOL>lib_jars,<EOL>extra_jars=[],<EOL>args=new_args<EOL>)<EOL>err_msg = "<STR_LIT>" % (action, topology_name)<EOL>succ_msg = "<STR_LIT>" % (action, topology_name)<EOL>result.add_context(err_msg, succ_msg)<EOL>return result<EOL>

helper function to take action on topologies
:param command:
:param cl_args:
:param action:        description of action taken
:return:


parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=True)<EOL>parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>nargs='<STR_LIT:?>',<EOL>default='<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>parser.set_defaults(subcommand='<STR_LIT>')<EOL>return parser<EOL>


def run(command, parser, args, unknown_args):

<EOL>command_help = args['<STR_LIT>']<EOL>if command_help == '<STR_LIT>':<EOL><INDENT>parser.print_help()<EOL>return SimpleResult(Status.Ok)<EOL><DEDENT>subparser = config.get_subparser(parser, command_help)<EOL>if subparser:<EOL><INDENT>print(subparser.format_help())<EOL>return SimpleResult(Status.Ok)<EOL><DEDENT>else:<EOL><INDENT>Log.error("<STR_LIT>", command_help)<EOL>return SimpleResult(Status.InvocationError)<EOL><DEDENT>

:param command:
:param parser:
:param args:
:param unknown_args:
:return:


return {<EOL>'<STR_LIT>': activate,<EOL>'<STR_LIT>': hconfig,<EOL>'<STR_LIT>': deactivate,<EOL>'<STR_LIT>': cli_help,<EOL>'<STR_LIT>': kill,<EOL>'<STR_LIT>': restart,<EOL>'<STR_LIT>': submit,<EOL>'<STR_LIT>': update,<EOL>'<STR_LIT:version>': version<EOL>}<EOL>

Create a map of command names and handlers


parser = argparse.ArgumentParser(<EOL>prog='<STR_LIT>',<EOL>epilog=HELP_EPILOG,<EOL>formatter_class=config.SubcommandHelpFormatter,<EOL>add_help=True)<EOL>subparsers = parser.add_subparsers(<EOL>title="<STR_LIT>",<EOL>metavar='<STR_LIT>')<EOL>command_list = sorted(command_handlers.items())<EOL>for command in command_list:<EOL><INDENT>command[<NUM_LIT:1>].create_parser(subparsers)<EOL><DEDENT>return parser<EOL>


if command in handlers:<EOL><INDENT>return handlers[command].run(command, parser, command_args, unknown_args)<EOL><DEDENT>else:<EOL><INDENT>err_context = '<STR_LIT>' % command<EOL>return result.SimpleResult(result.Status.InvocationError, err_context)<EOL><DEDENT>

Run the command
:param command:
:param parser:
:param command_args:
:param unknown_args:
:return:


for cur_file in files:<EOL><INDENT>if os.path.isdir(cur_file):<EOL><INDENT>shutil.rmtree(cur_file)<EOL><DEDENT>else:<EOL><INDENT>shutil.rmtree(os.path.dirname(cur_file))<EOL><DEDENT><DEDENT>


if not config.check_java_home_set():<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if not config.check_release_file_exists():<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

Check whether the environment variables are set
:return:


def server_deployment_mode(command, parser, cluster, cl_args):

<EOL>client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args)<EOL>if not client_confs[cluster]:<EOL><INDENT>return dict()<EOL><DEDENT>if not cl_args.get('<STR_LIT>', None):<EOL><INDENT>Log.debug("<STR_LIT>"% cliconfig.get_cluster_config_file(cluster))<EOL><DEDENT>else:<EOL><INDENT>Log.debug("<STR_LIT>" % cl_args['<STR_LIT>'])<EOL><DEDENT>if not '<STR_LIT>' in client_confs[cluster]:<EOL><INDENT>config_file = cliconfig.get_cluster_config_file(cluster)<EOL>Log.error('<STR_LIT>', cluster, config_file)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if '<STR_LIT>' in cl_args:<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>cluster_role_env = (cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>'])<EOL>config.server_mode_cluster_role_env(cluster_role_env, client_confs)<EOL>cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)<EOL><DEDENT>except Exception as ex:<EOL><INDENT>Log.error("<STR_LIT>", str(ex))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>new_cl_args = dict()<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:0>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:1>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:2>]<EOL>new_cl_args['<STR_LIT>'] = client_confs[cluster]['<STR_LIT>'].rstrip('<STR_LIT:/>')<EOL>new_cl_args['<STR_LIT>'] = config.SERVER_MODE<EOL>cl_args.update(new_cl_args)<EOL>return cl_args<EOL>

check the server deployment mode for the given cluster
if it is valid return the valid set of args
:param cluster:
:param cl_args:
:return:


def direct_deployment_mode(command, parser, cluster, cl_args):

cluster = cl_args['<STR_LIT>']<EOL>try:<EOL><INDENT>config_path = cl_args['<STR_LIT>']<EOL>override_config_file = config.parse_override_config_and_write_file(cl_args['<STR_LIT>'])<EOL><DEDENT>except KeyError:<EOL><INDENT>subparser = config.get_subparser(parser, command)<EOL>print(subparser.format_help())<EOL>return dict()<EOL><DEDENT>if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):<EOL><INDENT>Log.error("<STR_LIT>", config_path)<EOL>return dict()<EOL><DEDENT>config_path = config.get_heron_cluster_conf_dir(cluster, config_path)<EOL>if not os.path.isdir(config_path):<EOL><INDENT>Log.error("<STR_LIT>", config_path)<EOL>return dict()<EOL><DEDENT>Log.info("<STR_LIT>" % config_path)<EOL>try:<EOL><INDENT>cluster_role_env = (cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>'])<EOL>config.direct_mode_cluster_role_env(cluster_role_env, config_path)<EOL>cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)<EOL><DEDENT>except Exception as ex:<EOL><INDENT>Log.error("<STR_LIT>", str(ex))<EOL>return dict()<EOL><DEDENT>new_cl_args = dict()<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:0>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:1>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:2>]<EOL>new_cl_args['<STR_LIT>'] = config_path<EOL>new_cl_args['<STR_LIT>'] = override_config_file<EOL>new_cl_args['<STR_LIT>'] = config.DIRECT_MODE<EOL>cl_args.update(new_cl_args)<EOL>return cl_args<EOL>

check the direct deployment mode for the given cluster
if it is valid return the valid set of args
:param command:
:param parser:
:param cluster:
:param cl_args:
:return:


def extract_common_args(command, parser, cl_args):

try:<EOL><INDENT>cluster_role_env = cl_args.pop('<STR_LIT>')<EOL><DEDENT>except KeyError:<EOL><INDENT>try:<EOL><INDENT>cluster_role_env = cl_args.pop('<STR_LIT>')  <EOL><DEDENT>except KeyError:<EOL><INDENT>subparser = config.get_subparser(parser, command)<EOL>print(subparser.format_help())<EOL>return dict()<EOL><DEDENT><DEDENT>new_cl_args = dict()<EOL>cluster_tuple = config.get_cluster_role_env(cluster_role_env)<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:0>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:1>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:2>]<EOL>new_cl_args['<STR_LIT>'] = getpass.getuser()<EOL>cl_args.update(new_cl_args)<EOL>return cl_args<EOL>

Extract all the common args for all commands
:param command:
:param parser:
:param cl_args:
:return:


<EOL>check_environment()<EOL>parser = create_parser(handlers)<EOL>if len(sys.argv[<NUM_LIT:1>:]) == <NUM_LIT:0>:<EOL><INDENT>parser.print_help()<EOL>return <NUM_LIT:0><EOL><DEDENT>sys.argv = config.insert_bool_values(sys.argv)<EOL>try:<EOL><INDENT>args, unknown_args = parser.parse_known_args()<EOL><DEDENT>except ValueError as ex:<EOL><INDENT>Log.error("<STR_LIT>", str(ex))<EOL>Log.debug(traceback.format_exc())<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>command_line_args = vars(args)<EOL>log.set_logging_level(command_line_args)<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>mmand to be execute<EOL>command = command_line_args['<STR_LIT>']<EOL>is_local_command = command in local_commands<EOL>if command == '<STR_LIT:version>':<EOL><INDENT>results = run(handlers, command, parser, command_line_args, unknown_args)<EOL>return <NUM_LIT:0> if result.is_successful(results) else <NUM_LIT:1><EOL><DEDENT>if not is_local_command:<EOL><INDENT>log.set_logging_level(command_line_args)<EOL>Log.debug("<STR_LIT>", command_line_args)<EOL>command_line_args = extract_common_args(command, parser, command_line_args)<EOL>command_line_args = deployment_mode(command, parser, command_line_args)<EOL>if not command_line_args:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if command_line_args['<STR_LIT>'] == config.DIRECT_MODE and command != "<STR_LIT:version>":<EOL><INDENT>cleaned_up_files.append(command_line_args['<STR_LIT>'])<EOL>atexit.register(cleanup, cleaned_up_files)<EOL><DEDENT><DEDENT>Log.debug("<STR_LIT>", command_line_args)<EOL>start = time.time()<EOL>results = run(handlers, command, parser, command_line_args, unknown_args)<EOL>if not is_local_command:<EOL><INDENT>result.render(results)<EOL><DEDENT>end = time.time()<EOL>if not is_local_command:<EOL><INDENT>sys.stdout.flush()<EOL>Log.debug('<STR_LIT>', (end - start))<EOL><DEDENT>return <NUM_LIT:0> if result.is_successful(results) else <NUM_LIT:1><EOL>


def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):

<EOL>if extra_jars is None:<EOL><INDENT>extra_jars = []<EOL><DEDENT>if args is None:<EOL><INDENT>args = []<EOL><DEDENT>if java_defines is None:<EOL><INDENT>java_defines = []<EOL><DEDENT>java_opts = ['<STR_LIT>' + opt for opt in java_defines]<EOL>all_args = [config.get_java_path(), "<STR_LIT>", "<STR_LIT>"] +java_opts +["<STR_LIT>", config.get_classpath(extra_jars + lib_jars)]<EOL>all_args += [class_name] + list(args)<EOL>heron_env = os.environ.copy()<EOL>heron_env['<STR_LIT>'] = opts.get_heron_config()<EOL>Log.debug("<STR_LIT>", '<STR_LIT:U+0020>'.join(all_args))<EOL>Log.debug("<STR_LIT>", str(heron_env["<STR_LIT>"]))<EOL>process = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE, bufsize=<NUM_LIT:1>)<EOL>return ProcessResult(process)<EOL>

Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:


def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):

<EOL>tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='<STR_LIT>')<EOL>with contextlib.closing(tarfile.open(topology_tar)) as tar:<EOL><INDENT>tar.extractall(path=tmpdir)<EOL><DEDENT>topology_jar = os.path.basename(topology_tar).replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>") + "<STR_LIT>"<EOL>extra_jars = [<EOL>os.path.join(tmpdir, topology_jar),<EOL>os.path.join(tmpdir, "<STR_LIT:*>"),<EOL>os.path.join(tmpdir, "<STR_LIT>")<EOL>]<EOL>lib_jars = config.get_heron_libs(jars.topology_jars())<EOL>return heron_class(class_name, lib_jars, extra_jars, arguments, java_defines)<EOL>

:param class_name:
:param topology_tar:
:param arguments:
:param tmpdir_root:
:param java_defines:
:return:


return cli_helper.create_parser(subparsers, '<STR_LIT>', '<STR_LIT>')<EOL>


def run(command, parser, cl_args, unknown_args):

Log.debug("<STR_LIT>", cl_args)<EOL>return cli_helper.run(command, cl_args, "<STR_LIT>")<EOL>

:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:


def add_context(self, err_context, succ_context=None):

self.err_context = err_context<EOL>self.succ_context = succ_context<EOL>

Prepend msg to add some context information

        :param pmsg: context info
        :return: None


retcode = self.process.poll()<EOL>if retcode is not None and status_type(retcode) == Status.InvocationError:<EOL><INDENT>self._do_log(Log.error, stderr_line)<EOL><DEDENT>else:<EOL><INDENT>self._do_print(sys.stderr, stderr_line)<EOL><DEDENT>

render stderr of shelled-out process
            stderr could be error message of failure of invoking process or
            normal stderr output from successfully shelled-out process.
            In the first case, ``Popen'' should fail fast and we should be able to
            get return code immediately. We then render the failure message.
            In the second case, we simply print stderr line in stderr.
            The way to handle the first case is shaky but should be the best we can
            do since we have conflicts of design goals here.
        :param stderr_line: one line from shelled-out process
        :return:


<EOL>assert self.status is not None<EOL>if self.status == Status.Ok:<EOL><INDENT>self._do_log(Log.info, stdout)<EOL><DEDENT>elif self.status == Status.HeronError:<EOL><INDENT>self._do_log(Log.error, stdout)<EOL><DEDENT>elif self.status == Status.DryRun:<EOL><INDENT>self._do_print(sys.stdout, stdout)<EOL><DEDENT>elif self.status == Status.InvocationError:<EOL><INDENT>self._do_print(sys.stdout, stdout)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>" %(self.status.value, list(Status)))<EOL><DEDENT>

render stdout of shelled-out process
            stdout always contains information Java process wants to
            propagate back to cli, so we do special rendering here
        :param stdout: all lines from shelled-out process
        :return:


<EOL>parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>default=False,<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>)<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>default="<STR_LIT>",<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>action="<STR_LIT>",<EOL>default=[],<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>default="<STR_LIT>",<EOL>help='<STR_LIT>')<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>default="<STR_LIT>",<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>')<EOL>return parser<EOL>


default_format = '<STR_LIT>'<EOL>resp_formats = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>available_options = '<STR_LIT:U+002CU+0020>'.join(['<STR_LIT:%s>' % opt for opt in resp_formats])<EOL>def dry_run_resp_format(value):<EOL><INDENT>if value not in resp_formats:<EOL><INDENT>raise argparse.ArgumentTypeError(<EOL>'<STR_LIT>'<EOL>% (value, available_options))<EOL><DEDENT>return value<EOL><DEDENT>parser.add_argument(<EOL>'<STR_LIT>',<EOL>default=False,<EOL>action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>default='<STR_LIT>' if sys.stdout.isatty() else '<STR_LIT>',<EOL>type=dry_run_resp_format,<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>' % ('<STR_LIT:|>'.join(resp_formats), default_format))<EOL>return parser<EOL>


file_list = fnmatch.filter(os.listdir(dirname), pattern)<EOL>return file_list[<NUM_LIT:0>] if file_list else None<EOL>

Get the topology jars
:param dirname:
:param pattern:
:return:


jars = [<EOL>os.path.join(config.get_heron_lib_dir(), "<STR_LIT>", "<STR_LIT:*>")<EOL>]<EOL>return jars<EOL>


Log.debug("<STR_LIT>", cl_args)<EOL>container_id = cl_args['<STR_LIT>']<EOL>if cl_args['<STR_LIT>'] == config.SERVER_MODE:<EOL><INDENT>dict_extra_args = {"<STR_LIT>": str(container_id)}<EOL>return cli_helper.run_server(command, cl_args, "<STR_LIT>", extra_args=dict_extra_args)<EOL><DEDENT>else:<EOL><INDENT>list_extra_args = ["<STR_LIT>", str(container_id)]<EOL>return cli_helper.run_direct(command, cl_args, "<STR_LIT>", extra_args=list_extra_args)<EOL><DEDENT>


def run(command, parser, known_args, unknown_args):

config.print_build_info()<EOL>return True<EOL>


max_count = <NUM_LIT:20><EOL>table, count = [], <NUM_LIT:0><EOL>for role, envs_topos in result.items():<EOL><INDENT>for env, topos in envs_topos.items():<EOL><INDENT>for topo in topos:<EOL><INDENT>count += <NUM_LIT:1><EOL>if count > max_count:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>table.append([role, env, topo])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>header = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>rest_count = <NUM_LIT:0> if count <= max_count else count - max_count<EOL>return table, header, rest_count<EOL>


try:<EOL><INDENT>result = tracker_access.get_cluster_topologies(cluster)<EOL>if not result:<EOL><INDENT>Log.error('<STR_LIT>' % cluster)<EOL>return False<EOL><DEDENT>result = result[cluster]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>table, header, rest_count = to_table(result)<EOL>print('<STR_LIT>' % cluster)<EOL>if rest_count:<EOL><INDENT>print('<STR_LIT>' % rest_count)<EOL><DEDENT>print(tabulate(table, headers=header))<EOL>return True<EOL>


def show_cluster_role(cl_args, cluster, role):

try:<EOL><INDENT>result = tracker_access.get_cluster_role_topologies(cluster, role)<EOL>if not result:<EOL><INDENT>Log.error('<STR_LIT>' % '<STR_LIT:/>'.join([cluster, role]))<EOL>return False<EOL><DEDENT>result = result[cluster]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>table, header, rest_count = to_table(result)<EOL>print('<STR_LIT>' % (cluster, role))<EOL>if rest_count:<EOL><INDENT>print('<STR_LIT>' % rest_count)<EOL><DEDENT>print(tabulate(table, headers=header))<EOL>return True<EOL>


def show_cluster_role_env(cl_args, cluster, role, env):

try:<EOL><INDENT>result = tracker_access.get_cluster_role_env_topologies(cluster, role, env)<EOL>if not result:<EOL><INDENT>Log.error('<STR_LIT>' % '<STR_LIT:/>'.join([cluster, role, env]))<EOL>return False<EOL><DEDENT>result = result[cluster]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>table, header, rest_count = to_table(result)<EOL>print('<STR_LIT>' % (cluster, role, env))<EOL>if rest_count:<EOL><INDENT>print('<STR_LIT>' % rest_count)<EOL><DEDENT>print(tabulate(table, headers=header))<EOL>return True<EOL>


location = cl_args['<STR_LIT>'].split('<STR_LIT:/>')<EOL>if len(location) == <NUM_LIT:1>:<EOL><INDENT>return show_cluster(cl_args, *location)<EOL><DEDENT>elif len(location) == <NUM_LIT:2>:<EOL><INDENT>return show_cluster_role(cl_args, *location)<EOL><DEDENT>elif len(location) == <NUM_LIT:3>:<EOL><INDENT>return show_cluster_role_env(cl_args, *location)<EOL><DEDENT>else:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>


try:<EOL><INDENT>topo_loc = cl_args['<STR_LIT>'].split('<STR_LIT:/>')<EOL>topo_loc.append(cl_args['<STR_LIT>'])<EOL>if len(topo_loc) != <NUM_LIT:4>:<EOL><INDENT>raise<EOL><DEDENT>return topo_loc<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>raise<EOL><DEDENT>


inputs, outputs = defaultdict(list), defaultdict(list)<EOL>for ctype, component in components.items():<EOL><INDENT>if ctype == '<STR_LIT>':<EOL><INDENT>for component_name, component_info in component.items():<EOL><INDENT>for input_stream in component_info['<STR_LIT>']:<EOL><INDENT>input_name = input_stream['<STR_LIT>']<EOL>inputs[component_name].append(input_name)<EOL>outputs[input_name].append(component_name)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>info = []<EOL>spouts_instance = topo_info['<STR_LIT>']['<STR_LIT>']<EOL>bolts_instance = topo_info['<STR_LIT>']['<STR_LIT>']<EOL>for ctype, component in components.items():<EOL><INDENT>if ctype == "<STR_LIT>":<EOL><INDENT>continue<EOL><DEDENT>for component_name, component_info in component.items():<EOL><INDENT>row = [ctype[:-<NUM_LIT:1>], component_name]<EOL>if ctype == '<STR_LIT>':<EOL><INDENT>row.append(len(spouts_instance[component_name]))<EOL><DEDENT>else:<EOL><INDENT>row.append(len(bolts_instance[component_name]))<EOL><DEDENT>row.append('<STR_LIT:U+002C>'.join(inputs.get(component_name, ['<STR_LIT:->'])))<EOL>row.append('<STR_LIT:U+002C>'.join(outputs.get(component_name, ['<STR_LIT:->'])))<EOL>info.append(row)<EOL><DEDENT><DEDENT>header = ['<STR_LIT:type>', '<STR_LIT:name>', '<STR_LIT>', '<STR_LIT:input>', '<STR_LIT>']<EOL>return info, header<EOL>

normalize raw logical plan info to table


bolts_info = []<EOL>for row in table:<EOL><INDENT>if row[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>bolts_info.append(row)<EOL><DEDENT><DEDENT>return bolts_info, header<EOL>


spouts_info = []<EOL>for row in table:<EOL><INDENT>if row[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>spouts_info.append(row)<EOL><DEDENT><DEDENT>return spouts_info, header<EOL>


cluster, role, env = cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>']<EOL>topology = cl_args['<STR_LIT>']<EOL>spouts_only, bolts_only = cl_args['<STR_LIT>'], cl_args['<STR_LIT>']<EOL>try:<EOL><INDENT>components = tracker_access.get_logical_plan(cluster, env, topology, role)<EOL>topo_info = tracker_access.get_topology_info(cluster, env, topology, role)<EOL>table, header = to_table(components, topo_info)<EOL>if spouts_only == bolts_only:<EOL><INDENT>print(tabulate(table, headers=header))<EOL><DEDENT>elif spouts_only:<EOL><INDENT>table, header = filter_spouts(table, header)<EOL>print(tabulate(table, headers=header))<EOL><DEDENT>else:<EOL><INDENT>table, header = filter_bolts(table, header)<EOL>print(tabulate(table, headers=header))<EOL><DEDENT>return True<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>


def run_components(command, parser, cl_args, unknown_args):

return run(cl_args, '<STR_LIT:all>')<EOL>


def run_spouts(command, parser, cl_args, unknown_args):

return run(cl_args, '<STR_LIT>')<EOL>


def run_bolts(command, parser, cl_args, unknown_args):

return run(cl_args, '<STR_LIT>')<EOL>


define("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>


try:<EOL><INDENT>topo_loc = cl_args['<STR_LIT>'].split('<STR_LIT:/>')<EOL>topo_name = cl_args['<STR_LIT>']<EOL>topo_loc.append(topo_name)<EOL>if len(topo_loc) != <NUM_LIT:4>:<EOL><INDENT>raise<EOL><DEDENT>return topo_loc<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error('<STR_LIT>')<EOL>raise<EOL><DEDENT>


all_queries = tracker_access.metric_queries()<EOL>m = tracker_access.queries_map()<EOL>names = metrics.values()[<NUM_LIT:0>].keys()<EOL>stats = []<EOL>for n in names:<EOL><INDENT>info = [n]<EOL>for field in all_queries:<EOL><INDENT>try:<EOL><INDENT>info.append(str(metrics[field][n]))<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>stats.append(info)<EOL><DEDENT>header = ['<STR_LIT>'] + [m[k] for k in all_queries if k in metrics.keys()]<EOL>return stats, header<EOL>

normalize raw metrics API result to table


def run_metrics(command, parser, cl_args, unknown_args):

cluster, role, env = cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>']<EOL>topology = cl_args['<STR_LIT>']<EOL>try:<EOL><INDENT>result = tracker_access.get_topology_info(cluster, env, topology, role)<EOL>spouts = result['<STR_LIT>']['<STR_LIT>'].keys()<EOL>bolts = result['<STR_LIT>']['<STR_LIT>'].keys()<EOL>components = spouts + bolts<EOL>cname = cl_args['<STR_LIT>']<EOL>if cname:<EOL><INDENT>if cname in components:<EOL><INDENT>components = [cname]<EOL><DEDENT>else:<EOL><INDENT>Log.error('<STR_LIT>' % cname)<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>cresult = []<EOL>for comp in components:<EOL><INDENT>try:<EOL><INDENT>metrics = tracker_access.get_component_metrics(comp, cluster, env, topology, role)<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>stat, header = to_table(metrics)<EOL>cresult.append((comp, stat, header))<EOL><DEDENT>for i, (comp, stat, header) in enumerate(cresult):<EOL><INDENT>if i != <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>print('<STR_LIT>' % comp)<EOL>print(tabulate(stat, headers=header))<EOL><DEDENT>return True<EOL>


cluster, role, env = cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>']<EOL>topology = cl_args['<STR_LIT>']<EOL>try:<EOL><INDENT>result = tracker_access.get_topology_info(cluster, env, topology, role)<EOL>bolts = result['<STR_LIT>']['<STR_LIT>'].keys()<EOL>bolt_name = cl_args['<STR_LIT>']<EOL>if bolt_name:<EOL><INDENT>if bolt_name in bolts:<EOL><INDENT>bolts = [bolt_name]<EOL><DEDENT>else:<EOL><INDENT>Log.error('<STR_LIT>' % bolt_name)<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>bolts_result = []<EOL>for bolt in bolts:<EOL><INDENT>try:<EOL><INDENT>metrics = tracker_access.get_component_metrics(bolt, cluster, env, topology, role)<EOL>stat, header = to_table(metrics)<EOL>bolts_result.append((bolt, stat, header))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT><DEDENT>for i, (bolt, stat, header) in enumerate(bolts_result):<EOL><INDENT>if i != <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>print('<STR_LIT>' % bolt)<EOL>print(tabulate(stat, headers=header))<EOL><DEDENT>return True<EOL>


def run_containers(command, parser, cl_args, unknown_args):

cluster, role, env = cl_args['<STR_LIT>'], cl_args['<STR_LIT>'], cl_args['<STR_LIT>']<EOL>topology = cl_args['<STR_LIT>']<EOL>container_id = cl_args['<STR_LIT:id>']<EOL>try:<EOL><INDENT>result = tracker_access.get_topology_info(cluster, env, topology, role)<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>containers = result['<STR_LIT>']['<STR_LIT>']<EOL>all_bolts, all_spouts = set(), set()<EOL>for _, bolts in result['<STR_LIT>']['<STR_LIT>'].items():<EOL><INDENT>all_bolts = all_bolts | set(bolts)<EOL><DEDENT>for _, spouts in result['<STR_LIT>']['<STR_LIT>'].items():<EOL><INDENT>all_spouts = all_spouts | set(spouts)<EOL><DEDENT>stmgrs = containers.keys()<EOL>stmgrs.sort()<EOL>if container_id is not None:<EOL><INDENT>try:<EOL><INDENT>normalized_cid = container_id - <NUM_LIT:1><EOL>if normalized_cid < <NUM_LIT:0>:<EOL><INDENT>raise<EOL><DEDENT>stmgrs = [stmgrs[normalized_cid]]<EOL><DEDENT>except:<EOL><INDENT>Log.error('<STR_LIT>' % container_id)<EOL>return False<EOL><DEDENT><DEDENT>table = []<EOL>for sid, name in enumerate(stmgrs):<EOL><INDENT>cid = sid + <NUM_LIT:1><EOL>host = containers[name]["<STR_LIT:host>"]<EOL>port = containers[name]["<STR_LIT:port>"]<EOL>pid = containers[name]["<STR_LIT>"]<EOL>instances = containers[name]["<STR_LIT>"]<EOL>bolt_nums = len([instance for instance in instances if instance in all_bolts])<EOL>spout_nums = len([instance for instance in instances if instance in all_spouts])<EOL>table.append([cid, host, port, pid, bolt_nums, spout_nums, len(instances)])<EOL><DEDENT>headers = ["<STR_LIT>", "<STR_LIT:host>", "<STR_LIT:port>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>sys.stdout.flush()<EOL>print(tabulate(table, headers=headers))<EOL>return True<EOL>


parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=False)<EOL>parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>nargs='<STR_LIT:?>',<EOL>default='<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>parser.set_defaults(subcommand='<STR_LIT>')<EOL>return parser<EOL>


<EOL>command_help = args['<STR_LIT>']<EOL>if command_help == '<STR_LIT>':<EOL><INDENT>parser.print_help()<EOL>return True<EOL><DEDENT>subparser = config.get_subparser(parser, command_help)<EOL>if subparser:<EOL><INDENT>print(subparser.format_help())<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>Log.error("<STR_LIT>" % command_help)<EOL>return False<EOL><DEDENT>


<EOL>if command == '<STR_LIT>':<EOL><INDENT>return clusters.run(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return topologies.run(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return physicalplan.run_containers(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return physicalplan.run_metrics(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return logicalplan.run_components(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return logicalplan.run_spouts(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return logicalplan.run_bolts(command, *args)<EOL><DEDENT>elif command == '<STR_LIT>':<EOL><INDENT>return help.run(command, *args)<EOL><DEDENT>elif command == '<STR_LIT:version>':<EOL><INDENT>return version.run(command, *args)<EOL><DEDENT>return <NUM_LIT:1><EOL>


try:<EOL><INDENT>cluster_role_env = cl_args['<STR_LIT>']<EOL>config_path = cl_args['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>subparser = config.get_subparser(parser, command)<EOL>print(subparser.format_help())<EOL>return dict()<EOL><DEDENT>cluster = config.get_heron_cluster(cluster_role_env)<EOL>config_path = config.get_heron_cluster_conf_dir(cluster, config_path)<EOL>new_cl_args = dict()<EOL>try:<EOL><INDENT>cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:0>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:1>]<EOL>new_cl_args['<STR_LIT>'] = cluster_tuple[<NUM_LIT:2>]<EOL>new_cl_args['<STR_LIT>'] = config_path<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>", str(e))<EOL>return dict()<EOL><DEDENT>cl_args.update(new_cl_args)<EOL>return cl_args<EOL>


<EOL>parser = create_parser()<EOL>if not args:<EOL><INDENT>parser.print_help()<EOL>return <NUM_LIT:0><EOL><DEDENT>all_args = parse.insert_bool_values(args)<EOL>args, unknown_args = parser.parse_known_args(args=all_args)<EOL>command_line_args = vars(args)<EOL>command = command_line_args['<STR_LIT>']<EOL>if unknown_args:<EOL><INDENT>Log.error('<STR_LIT>', unknown_args[<NUM_LIT:0>])<EOL>command_line_args['<STR_LIT>'] = command<EOL>command = '<STR_LIT>'<EOL><DEDENT>if command not in ['<STR_LIT>', '<STR_LIT:version>']:<EOL><INDENT>opts.set_tracker_url(command_line_args)<EOL>log.set_logging_level(command_line_args)<EOL>if command not in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>command_line_args = extract_common_args(command, parser, command_line_args)<EOL><DEDENT>if not command_line_args:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>Log.info("<STR_LIT>", command_line_args["<STR_LIT>"])<EOL><DEDENT>start = time.time()<EOL>ret = run(command, parser, command_line_args, unknown_args)<EOL>end = time.time()<EOL>if command != '<STR_LIT>':<EOL><INDENT>sys.stdout.flush()<EOL>Log.info('<STR_LIT>', (end - start))<EOL><DEDENT>return <NUM_LIT:0> if ret else <NUM_LIT:1><EOL>


try:<EOL><INDENT>clusters = tracker_access.get_clusters()<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>", cl_args["<STR_LIT>"])<EOL>return False<EOL><DEDENT>print('<STR_LIT>')<EOL>for cluster in clusters:<EOL><INDENT>print('<STR_LIT>' % cluster)<EOL><DEDENT>return True<EOL>


<EOL>default_config_path = config.get_heron_conf_dir()<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + default_config_path + '<STR_LIT>',<EOL>default=os.path.join(config.get_heron_dir(), default_config_path))<EOL>return parser<EOL>


parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>return parser<EOL>


index = <NUM_LIT:0><EOL>found = False<EOL>for lelem in command_args:<EOL><INDENT>if lelem == '<STR_LIT>' and not found:<EOL><INDENT>break<EOL><DEDENT>if lelem == param:<EOL><INDENT>found = True<EOL>break<EOL><DEDENT>index = index + <NUM_LIT:1><EOL><DEDENT>if found:<EOL><INDENT>command_args.insert(index + <NUM_LIT:1>, '<STR_LIT:True>')<EOL><DEDENT>return command_args<EOL>


args1 = insert_bool('<STR_LIT>', command_line_args)<EOL>return args1<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>type=bool,<EOL>default=False)<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + DEFAULT_TRACKER_URL + '<STR_LIT>',<EOL>type=str, default=DEFAULT_TRACKER_URL)<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>type=int, metavar='<STR_LIT>')<EOL>return parser<EOL>

add optional argument that specifies container id


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>metavar='<STR_LIT>',<EOL>type=str)<EOL>return parser<EOL>

add optional argument that specifies component name


parser.add_argument(<EOL>'<STR_LIT>', help='<STR_LIT>', action='<STR_LIT:store_true>')<EOL>return parser<EOL>

add optional argument that displays spout only


add optional argument that displays bolts only


parser.add_argument(<EOL>'<STR_LIT>', help='<STR_LIT>', type=str,<EOL>metavar='<STR_LIT>')<EOL>return parser<EOL>

add argument that specifies topologies location


parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>)<EOL>return parser<EOL>

add argument that specifies topology name


def get(self, cluster, environ, topology):

:param cluster:
:param environ:
:param topology:
:return:


def get(self, cluster, environ, topology, comp_name, instance):

:param cluster:
:param environ:
:param topology:
:param comp_name:
:param instance:
:return:


clusters = yield access.get_clusters()<EOL>options = dict(<EOL>topologies=[],  <EOL>clusters=[str(cluster) for cluster in clusters],<EOL>active="<STR_LIT>",  <EOL>function=common.className,<EOL>baseUrl=self.baseUrl<EOL>)<EOL>self.render("<STR_LIT>", **options)<EOL>


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology):<DEDENT>

<EOL>execution_state = yield access.get_execution_state(cluster, environ, topology)<EOL>scheduler_location = yield access.get_scheduler_location(cluster, environ, topology)<EOL>job_page_link = scheduler_location["<STR_LIT>"]<EOL>launched_at = datetime.utcfromtimestamp(execution_state['<STR_LIT>'])<EOL>launched_time = launched_at.strftime('<STR_LIT>')<EOL>options = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>execution_state=execution_state,<EOL>launched=launched_time,<EOL>status="<STR_LIT>" if random.randint(<NUM_LIT:0>, <NUM_LIT:1>) else "<STR_LIT>",<EOL>active="<STR_LIT>",<EOL>job_page_link=job_page_link,<EOL>function=common.className,<EOL>baseUrl=self.baseUrl<EOL>)<EOL>self.render("<STR_LIT>", **options)<EOL>


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology, container):<DEDENT>

path = self.get_argument("<STR_LIT:path>")<EOL>options = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path,<EOL>baseUrl=self.baseUrl<EOL>)<EOL>self.render("<STR_LIT>", **options)<EOL>

:param cluster:
:param environ:
:param topology:
:param container:
:return:


offset = self.get_argument("<STR_LIT>")<EOL>length = self.get_argument("<STR_LIT>")<EOL>path = self.get_argument("<STR_LIT:path>")<EOL>data = yield access.get_container_file_data(cluster, environ, topology, container, path,<EOL>offset, length)<EOL>self.write(data)<EOL>self.finish()<EOL>


path = self.get_argument("<STR_LIT:path>", default="<STR_LIT:.>")<EOL>data = yield access.get_filestats(cluster, environ, topology, container, path)<EOL>options = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path,<EOL>filestats=data,<EOL>baseUrl=self.baseUrl)<EOL>self.render("<STR_LIT>", **options)<EOL>