<EOL>self.connection_closed = False<EOL>path = self.get_argument("<STR_LIT:path>")<EOL>filename = path.split("<STR_LIT:/>")[-<NUM_LIT:1>]<EOL>self.set_header("<STR_LIT>", "<STR_LIT>" % filename)<EOL>file_download_url = access.get_container_file_download_url(cluster, environ,<EOL>topology, container, path)<EOL>Log.debug("<STR_LIT>", str(file_download_url))<EOL>def streaming_callback(chunk):<EOL><INDENT>self.write(chunk)<EOL>self.flush()<EOL><DEDENT>http_client = tornado.httpclient.AsyncHTTPClient()<EOL>yield http_client.fetch(file_download_url, streaming_callback=streaming_callback)<EOL>self.finish()<EOL>


<EOL>self.connection_closed = True<EOL>


errormessage = "<STR_LIT>"<EOL>self.render("<STR_LIT>", errormessage=errormessage)<EOL>


def write_error(self, status_code, **kwargs):

if "<STR_LIT>" in kwargs:<EOL><INDENT>exc_info = kwargs["<STR_LIT>"]<EOL>error = exc_info[<NUM_LIT:1>]<EOL>errormessage = "<STR_LIT>" % (status_code, error)<EOL>self.render("<STR_LIT>", errormessage=errormessage)<EOL><DEDENT>else:<EOL><INDENT>errormessage = "<STR_LIT:%s>" % (status_code)<EOL>self.render("<STR_LIT>", errormessage=errormessage)<EOL><DEDENT>

:param status_code:
:param kwargs:
:return:


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology, comp_name):<DEDENT>

start_time = time.time()<EOL>comp_names = []<EOL>if comp_name == "<STR_LIT>":<EOL><INDENT>lplan = yield access.get_logical_plan(cluster, environ, topology)<EOL>if not lplan:<EOL><INDENT>self.write(dict())<EOL>return<EOL><DEDENT>if not '<STR_LIT>' in lplan or not '<STR_LIT>' in lplan:<EOL><INDENT>self.write(dict())<EOL>return<EOL><DEDENT>comp_names = lplan['<STR_LIT>'].keys()<EOL>comp_names.extend(lplan['<STR_LIT>'].keys())<EOL><DEDENT>else:<EOL><INDENT>comp_names = [comp_name]<EOL><DEDENT>exception_infos = dict()<EOL>for comp_name in comp_names:<EOL><INDENT>exception_infos[comp_name] = yield access.get_component_exceptionsummary(<EOL>cluster, environ, topology, comp_name)<EOL><DEDENT>aggregate_exceptions = dict()<EOL>for comp_name, exception_logs in exception_infos.items():<EOL><INDENT>for exception_log in exception_logs:<EOL><INDENT>class_name = exception_log['<STR_LIT>']<EOL>if class_name != '<STR_LIT>':<EOL><INDENT>if not class_name in aggregate_exceptions:<EOL><INDENT>aggregate_exceptions[class_name] = <NUM_LIT:0><EOL><DEDENT>aggregate_exceptions[class_name] += int(exception_log['<STR_LIT:count>'])<EOL><DEDENT><DEDENT><DEDENT>aggregate_exceptions_table = []<EOL>for key in aggregate_exceptions:<EOL><INDENT>aggregate_exceptions_table.append([key, str(aggregate_exceptions[key])])<EOL><DEDENT>result = dict(<EOL>status="<STR_LIT:success>",<EOL>executiontime=time.time() - start_time,<EOL>result=aggregate_exceptions_table)<EOL>self.write(result)<EOL>

:param cluster:
:param environ:
:param topology:
:param comp_name:
:return:


<EOL>topologies = yield access.get_topologies_states()<EOL>result = dict()<EOL>for cluster, cluster_value in topologies.items():<EOL><INDENT>result[cluster] = dict()<EOL>for environ, environ_value in cluster_value.items():<EOL><INDENT>result[cluster][environ] = dict()<EOL>for topology, topology_value in environ_value.items():<EOL><INDENT>if "<STR_LIT>" not in topology_value or topology_value["<STR_LIT>"] is None:<EOL><INDENT>continue<EOL><DEDENT>if "<STR_LIT>" in topology_value:<EOL><INDENT>topology_value["<STR_LIT>"] = topology_value["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>topology_value["<STR_LIT>"] = '<STR_LIT:->'<EOL><DEDENT>result[cluster][environ][topology] = topology_value<EOL><DEDENT><DEDENT><DEDENT>self.write(result)<EOL>


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology):<DEDENT>

start_time = time.time()<EOL>lplan = yield access.get_logical_plan(cluster, environ, topology)<EOL>result = dict(<EOL>status="<STR_LIT:success>",<EOL>message="<STR_LIT>",<EOL>version=common.VERSION,<EOL>executiontime=time.time() - start_time,<EOL>result=lplan<EOL>)<EOL>self.write(result)<EOL>

:param cluster:
:param environ:
:param topology:
:return:


start_time = time.time()<EOL>pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>result_map = dict(<EOL>status="<STR_LIT:success>",<EOL>message="<STR_LIT>",<EOL>version=common.VERSION,<EOL>executiontime=time.time() - start_time,<EOL>result=pplan<EOL>)<EOL>self.write(result_map)<EOL>


start_time = time.time()<EOL>estate = yield access.get_execution_state(cluster, environ, topology)<EOL>result_map = dict(<EOL>status="<STR_LIT:success>",<EOL>message="<STR_LIT>",<EOL>version=common.VERSION,<EOL>executiontime=time.time() - start_time,<EOL>result=estate<EOL>)<EOL>self.write(result_map)<EOL>


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology, component):<DEDENT>

start_time = time.time()<EOL>futures = yield access.get_component_exceptions(cluster, environ, topology, component)<EOL>result_map = dict(<EOL>status='<STR_LIT:success>',<EOL>executiontime=time.time() - start_time,<EOL>result=futures)<EOL>self.write(json.dumps(result_map))<EOL>

:param cluster:
:param environ:
:param topology:
:param component:
:return:


@tornado.gen.coroutine<EOL><INDENT>def get(self, cluster, environ, topology, instance):<DEDENT>

pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.get_instance_pid(<EOL>cluster, environ, topology, instance)))<EOL>self.write('<STR_LIT>' % (<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>

:param cluster:
:param environ:
:param topology:
:param instance:
:return:


pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.get_instance_jstack(<EOL>cluster, environ, topology, instance)))<EOL>self.write('<STR_LIT>' % (<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>


pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.get_instance_mem_histogram(<EOL>cluster, environ, topology, instance)))<EOL>self.write('<STR_LIT>' % (<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>


pplan = yield access.get_physical_plan(cluster, environ, topology)<EOL>host = pplan['<STR_LIT>'][pplan['<STR_LIT>'][instance]['<STR_LIT>']]['<STR_LIT:host>']<EOL>result = json.loads((yield access.run_instance_jmap(<EOL>cluster, environ, topology, instance)))<EOL>notes = "<STR_LIT>".join([<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % host<EOL>])<EOL>self.write('<STR_LIT>' % (<EOL>notes,<EOL>host,<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>']),<EOL>tornado.escape.xhtml_escape(result['<STR_LIT>'])))<EOL>


cluster = self.get_argument("<STR_LIT>")<EOL>environ = self.get_argument("<STR_LIT>")<EOL>topology = self.get_argument("<STR_LIT>")<EOL>component = self.get_argument("<STR_LIT>", default=None)<EOL>metricnames = self.get_arguments("<STR_LIT>")<EOL>instances = self.get_arguments("<STR_LIT>")<EOL>interval = self.get_argument("<STR_LIT>", default=-<NUM_LIT:1>)<EOL>time_range = (<NUM_LIT:0>, interval)<EOL>compnames = [component] if component else (yield access.get_comps(cluster, environ, topology))<EOL>futures = {}<EOL>for comp in compnames:<EOL><INDENT>future = access.get_comp_metrics(<EOL>cluster, environ, topology, comp, instances,<EOL>metricnames, time_range)<EOL>futures[comp] = future<EOL><DEDENT>results = yield futures<EOL>self.write(results[component] if component else results)<EOL>


cluster = self.get_argument("<STR_LIT>")<EOL>environ = self.get_argument("<STR_LIT>")<EOL>topology = self.get_argument("<STR_LIT>")<EOL>component = self.get_argument("<STR_LIT>", default=None)<EOL>metric = self.get_argument("<STR_LIT>")<EOL>instances = self.get_argument("<STR_LIT>")<EOL>start = self.get_argument("<STR_LIT>")<EOL>end = self.get_argument("<STR_LIT>")<EOL>maxquery = self.get_argument("<STR_LIT>", default=False)<EOL>timerange = (start, end)<EOL>compnames = [component]<EOL>futures = {}<EOL>if metric == "<STR_LIT>":<EOL><INDENT>for comp in compnames:<EOL><INDENT>future = query_handler.fetch_backpressure(cluster, metric, topology, component,<EOL>instances, timerange, maxquery, environ)<EOL>futures[comp] = future<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fetch = query_handler.fetch_max if maxquery else query_handler.fetch<EOL>for comp in compnames:<EOL><INDENT>future = fetch(cluster, metric, topology, component,<EOL>instances, timerange, environ)<EOL>futures[comp] = future<EOL><DEDENT><DEDENT>results = yield futures<EOL>self.write(results[component] if component else results)<EOL>


<EOL>now = int(time.time())<EOL>time_slots = dict()<EOL>for key, value in ranges.items():<EOL><INDENT>time_slots[key] = (now - value[<NUM_LIT:0>], now - value[<NUM_LIT:1>], value[<NUM_LIT:2>])<EOL><DEDENT>return (now, time_slots)<EOL>


if selected == item:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>return '<STR_LIT>'<EOL>


def define_options(address, port, tracker_url, base_url):

define("<STR_LIT:address>", default=address)<EOL>define("<STR_LIT:port>", default=port)<EOL>define("<STR_LIT>", default=tracker_url)<EOL>define("<STR_LIT>", default=base_url)<EOL>

:param address:
:param port:
:param tracker_url:
:return:


log.configure(logging.DEBUG)<EOL>tornado.log.enable_pretty_logging()<EOL>(parser, child_parser) = args.create_parsers()<EOL>(parsed_args, remaining) = parser.parse_known_args()<EOL>if remaining:<EOL><INDENT>r = child_parser.parse_args(args=remaining, namespace=parsed_args)<EOL>namespace = vars(r)<EOL>if '<STR_LIT:version>' in namespace:<EOL><INDENT>common_config.print_build_info(zipped_pex=True)<EOL><DEDENT>else:<EOL><INDENT>parser.print_help()<EOL><DEDENT>parser.exit()<EOL><DEDENT>command_line_args = vars(parsed_args)<EOL>Log.info("<STR_LIT>", command_line_args['<STR_LIT:address>'],<EOL>command_line_args['<STR_LIT:port>'], command_line_args['<STR_LIT>'])<EOL>Log.info("<STR_LIT>", command_line_args['<STR_LIT>'])<EOL>define_options(command_line_args['<STR_LIT:address>'],<EOL>command_line_args['<STR_LIT:port>'],<EOL>command_line_args['<STR_LIT>'],<EOL>command_line_args['<STR_LIT>'])<EOL>http_server = tornado.httpserver.HTTPServer(Application(command_line_args['<STR_LIT>']))<EOL>http_server.listen(command_line_args['<STR_LIT:port>'], address=command_line_args['<STR_LIT:address>'])<EOL>def signal_handler(signum, frame):<EOL><INDENT>print('<STR_LIT:\n>', end='<STR_LIT>')<EOL>Log.debug('<STR_LIT>')<EOL>tornado.ioloop.IOLoop.instance().stop()<EOL><DEDENT>signal.signal(signal.SIGINT, signal_handler)<EOL>signal.signal(signal.SIGTERM, signal_handler)<EOL>tornado.ioloop.IOLoop.instance().start()<EOL>


parser._positionals.title = "<STR_LIT>"<EOL>parser._optionals.title = "<STR_LIT>"<EOL>return parser<EOL>


parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + consts.DEFAULT_TRACKER_URL + '<STR_LIT>',<EOL>default=consts.DEFAULT_TRACKER_URL)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + consts.DEFAULT_ADDRESS + '<STR_LIT>',<EOL>default=consts.DEFAULT_ADDRESS)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>' + str(consts.DEFAULT_PORT) + '<STR_LIT:)>',<EOL>type=int,<EOL>default=consts.DEFAULT_PORT)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>metavar='<STR_LIT>'<EOL>+ str(consts.DEFAULT_BASE_URL) + '<STR_LIT:)>',<EOL>default=consts.DEFAULT_BASE_URL)<EOL>return parser<EOL>


parser = argparse.ArgumentParser(<EOL>epilog='<STR_LIT>',<EOL>usage="<STR_LIT>",<EOL>add_help=False)<EOL>parser = add_titles(parser)<EOL>parser = add_arguments(parser)<EOL>child_parser = argparse.ArgumentParser(<EOL>parents=[parser],<EOL>formatter_class=SubcommandHelpFormatter,<EOL>add_help=False)<EOL>subparsers = child_parser.add_subparsers(<EOL>title="<STR_LIT>")<EOL>help_parser = subparsers.add_parser(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>',<EOL>add_help=False)<EOL>version_parser = subparsers.add_parser(<EOL>'<STR_LIT:version>',<EOL>help='<STR_LIT>',<EOL>add_help=True)<EOL>help_parser.set_defaults(help=True)<EOL>version_parser.set_defaults(version=True)<EOL>return (parser, child_parser)<EOL>


qs = _all_metric_queries()<EOL>return qs[<NUM_LIT:0>] + qs[<NUM_LIT:2>]<EOL>


qs = _all_metric_queries()<EOL>return dict(zip(qs[<NUM_LIT:0>], qs[<NUM_LIT:1>]) + zip(qs[<NUM_LIT:2>], qs[<NUM_LIT:3>]))<EOL>


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_clusters())<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get all cluster names


def get_logical_plan(cluster, env, topology, role):

instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_topology_info(*args))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topology information


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_comp_metrics(*args))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>


def get_component_metrics(component, cluster, env, topology, role):

all_queries = metric_queries()<EOL>try:<EOL><INDENT>result = get_topology_metrics(cluster, env, topology, component, [],<EOL>all_queries, [<NUM_LIT:0>, -<NUM_LIT:1>], role)<EOL>return result["<STR_LIT>"]<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get component metrics


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_cluster_topologies(cluster))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topologies under a cluster


instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_cluster_role_topologies(cluster, role))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topologies under a cluster submitted by a role


def get_cluster_role_env_topologies(cluster, role, env):

instance = tornado.ioloop.IOLoop.instance()<EOL>try:<EOL><INDENT>return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env))<EOL><DEDENT>except Exception:<EOL><INDENT>Log.debug(traceback.format_exc())<EOL>raise<EOL><DEDENT>

Synced API call to get topologies under a cluster submitted by a role under env


def fetch(self, cluster, metric, topology, component, instance, timerange, envirn=None):

pass<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param envirn:
:return:


def fetch_max(self, cluster, metric, topology, component, instance, timerange, envirn=None):

pass<EOL>


def fetch_backpressure(self, cluster, metric, topology, component, instance,timerange, is_max, environ=None):

pass<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param is_max:
:param environ:
:return:


<EOL>if default_value is None:<EOL><INDENT>default_value = dict()<EOL><DEDENT>Log.debug("<STR_LIT>", fetch_url)<EOL>ret = default_value<EOL>start = time.time()<EOL>http_response = yield tornado.httpclient.AsyncHTTPClient().fetch(fetch_url)<EOL>if http_response.error:<EOL><INDENT>Log.error("<STR_LIT>", fetch_url, http_response.error)<EOL>raise tornado.gen.Return(ret)<EOL><DEDENT>response = json.loads(http_response.body)<EOL>if not '<STR_LIT:result>' in response:<EOL><INDENT>Log.error("<STR_LIT>", fetch_url)<EOL>raise tornado.gen.Return(ret)<EOL><DEDENT>ret = response['<STR_LIT:result>']<EOL>execution = <NUM_LIT:1000> * response['<STR_LIT>']<EOL>end = time.time()<EOL>duration = <NUM_LIT:1000> * (end - start)<EOL>Log.debug("<STR_LIT>", execution, fetch_url)<EOL>Log.debug("<STR_LIT>", duration, fetch_url)<EOL>raise tornado.gen.Return(ret)<EOL>

Fetch the given url and convert the response to json.
:param fetch_url: URL to fetch
:param default_value: value to return in case of failure
:return:


return options.tracker_url<EOL>

Get the endpoint for heron tracker
:return:


return fmt % get_tracker_endpoint()<EOL>

Given an URL format, substitute with tracker service endpoint
:param fmt:
:return:


request_url = create_url(CLUSTER_URL_FMT)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


request_url = create_url(TOPOLOGIES_URL_FMT)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the list of topologies given a data center from heron tracker
:return:


request_url = create_url(TOPOLOGIES_STATS_URL_FMT)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the list of topologies and their states
:return:


return _get_topologies(cluster)<EOL>

Get the list of topologies given a cluster
:param cluster:
:return:


return _get_topologies(cluster, role=role)<EOL>

Get the list of topologies given a cluster submitted by a given role
:param cluster:
:param role:
:return:


return _get_topologies(cluster, role=role, env=env)<EOL>

Get the list of topologies given a cluster submitted by a given role under a given environment
:param cluster:
:param role:
:param env:
:return:


@tornado.gen.coroutine<EOL>def get_execution_state(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(EXECUTION_STATE_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the execution state of a topology in a cluster
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_logical_plan(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(LOGICALPLAN_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the logical plan state of a topology in a cluster
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_comps(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(LOGICALPLAN_URL_FMT), params)<EOL>lplan = yield fetch_url_as_json(request_url)<EOL>comps = lplan['<STR_LIT>'].keys() + lplan['<STR_LIT>'].keys()<EOL>raise tornado.gen.Return(comps)<EOL>

Get the list of component names for the topology from Heron Nest
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_instances(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(PHYSICALPLAN_URL_FMT), params)<EOL>pplan = yield fetch_url_as_json(request_url)<EOL>instances = pplan['<STR_LIT>'].keys()<EOL>raise tornado.gen.Return(instances)<EOL>

Get the list of instances for the topology from Heron Nest
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_physical_plan(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(PHYSICALPLAN_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the physical plan state of a topology in a cluster from tracker
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_scheduler_location(cluster, environ, topology, role=None):

params = dict(cluster=cluster, environ=environ, topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(SCHEDULER_LOCATION_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the scheduler location of a topology in a cluster from tracker
:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_component_exceptionsummary(cluster, environ, topology, component, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(EXCEPTION_SUMMARY_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get summary of exception for a component
:param cluster:
:param environ:
:param topology:
:param component:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_component_exceptions(cluster, environ, topology, component, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(EXCEPTIONS_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get exceptions for 'component' for 'topology'
:param cluster:
:param environ:
:param topology:
:param component:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_comp_instance_metrics(cluster, environ, topology, component,<EOL>metrics, instances, time_range, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(METRICS_URL_FMT), params)<EOL>all_instances = instances if isinstance(instances, list) else [instances]<EOL>for _, metric_name in metrics.items():<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name[<NUM_LIT:0>]))<EOL><DEDENT>for i in all_instances:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(instance=i))<EOL><DEDENT>request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[<NUM_LIT:1>]))<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the metrics for some instances of a topology from tracker
:param cluster:
:param environ:
:param topology:
:param component:
:param metrics:     dict of display name to cuckoo name
:param instances:
:param time_range:  2-tuple consisting of start and end of range
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_comp_metrics(cluster, environ, topology, component,<EOL>instances, metricnames, time_range, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(METRICS_URL_FMT), params)<EOL>for metric_name in metricnames:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))<EOL><DEDENT>for instance in instances:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))<EOL><DEDENT>request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[<NUM_LIT:1>]))<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the metrics for all the instances of a topology from Heron Nest
:param cluster:
:param environ:
:param topology:
:param component:
:param instances:
:param metricnames:   dict of display name to cuckoo name
:param time_range:    2-tuple consisting of start and end of range
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_metrics(cluster, environment, topology, timerange, query, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environment,<EOL>topology=topology,<EOL>starttime=timerange[<NUM_LIT:0>],<EOL>endtime=timerange[<NUM_LIT:1>],<EOL>query=query)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(METRICS_QUERY_URL_FMT), params<EOL>)<EOL>logging.info("<STR_LIT>", request_url)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the metrics for a topology from tracker
:param cluster:
:param environment:
:param topology:
:param timerange:
:param query:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_comp_metrics_timeline(cluster, environ, topology, component,<EOL>instances, metricnames, time_range, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>component=component)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(METRICS_TIMELINE_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>for metric_name in metricnames:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))<EOL><DEDENT>for instance in instances:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>request_url, dict(starttime=time_range[<NUM_LIT:0>], endtime=time_range[<NUM_LIT:1>]))<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

Get the minute-by-minute metrics for all instances of a topology from tracker
:param cluster:
:param environ:
:param topology:
:param component:
:param instances:
:param metricnames:   dict of display name to cuckoo name
:param time_range:    2-tuple consisting of start and end of range
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_topology_info(cluster, environ, topology, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(INFO_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

:param cluster:
:param environ:
:param topology:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_instance_pid(cluster, environ, topology, instance, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>instance=instance)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(PID_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

:param cluster:
:param environ:
:param topology:
:param instance:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_instance_jstack(cluster, environ, topology, instance, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>instance=instance)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(JSTACK_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


@tornado.gen.coroutine<EOL>def get_instance_mem_histogram(cluster, environ, topology, instance, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>instance=instance)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(HISTOGRAM_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


@tornado.gen.coroutine<EOL>def run_instance_jmap(cluster, environ, topology, instance, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>instance=instance)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(JMAP_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


def get_container_file_download_url(cluster, environ, topology, container,<EOL>path, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(FILE_DOWNLOAD_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>return request_url<EOL>

:param cluster:
:param environ:
:param topology:
:param container:
:param path:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_container_file_data(cluster, environ, topology, container,<EOL>path, offset, length, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path,<EOL>offset=offset,<EOL>length=length)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(<EOL>create_url(FILE_DATA_URL_FMT), params)<EOL>if role is not None:<EOL><INDENT>request_url = tornado.httputil.url_concat(request_url, dict(role=role))<EOL><DEDENT>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>

:param cluster:
:param environ:
:param topology:
:param container:
:param path:
:param offset:
:param length:
:param role:
:return:


@tornado.gen.coroutine<EOL>def get_filestats(cluster, environ, topology, container, path, role=None):

params = dict(<EOL>cluster=cluster,<EOL>environ=environ,<EOL>topology=topology,<EOL>container=container,<EOL>path=path)<EOL>if role is not None:<EOL><INDENT>params['<STR_LIT>'] = role<EOL><DEDENT>request_url = tornado.httputil.url_concat(create_url(FILESTATS_URL_FMT), params)<EOL>raise tornado.gen.Return((yield fetch_url_as_json(request_url)))<EOL>


@tornado.gen.coroutine<EOL><INDENT>def fetch(self, cluster, metric, topology, component, instance, timerange, environ=None):<DEDENT>

components = [component] if component != "<STR_LIT:*>" else (yield get_comps(cluster, environ, topology))<EOL>futures = []<EOL>for comp in components:<EOL><INDENT>query = self.get_query(metric, comp, instance)<EOL>future = get_metrics(cluster, environ, topology, timerange, query)<EOL>futures.append(future)<EOL><DEDENT>results = yield futures<EOL>timelines = []<EOL>for result in results:<EOL><INDENT>timelines.extend(result["<STR_LIT>"])<EOL><DEDENT>result = self.get_metric_response(timerange, timelines, False)<EOL>raise tornado.gen.Return(result)<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param environ:
:return:


@tornado.gen.coroutine<EOL><INDENT>def fetch_max(self, cluster, metric, topology, component, instance, timerange, environ=None):<DEDENT>

components = [component] if component != "<STR_LIT:*>" else (yield get_comps(cluster, environ, topology))<EOL>result = {}<EOL>futures = []<EOL>for comp in components:<EOL><INDENT>query = self.get_query(metric, comp, instance)<EOL>max_query = "<STR_LIT>" % query<EOL>future = get_metrics(cluster, environ, topology, timerange, max_query)<EOL>futures.append(future)<EOL><DEDENT>results = yield futures<EOL>data = self.compute_max(results)<EOL>result = self.get_metric_response(timerange, data, True)<EOL>raise tornado.gen.Return(result)<EOL>


@tornado.gen.coroutine<EOL><INDENT>def fetch_backpressure(self, cluster, metric, topology, component, instance,timerange, is_max, environ=None):<DEDENT>

instances = yield get_instances(cluster, environ, topology)<EOL>if component != "<STR_LIT:*>":<EOL><INDENT>filtered_inst = [instance for instance in instances if instance.split("<STR_LIT:_>")[<NUM_LIT:2>] == component]<EOL><DEDENT>else:<EOL><INDENT>filtered_inst = instances<EOL><DEDENT>futures_dict = {}<EOL>for inst in filtered_inst:<EOL><INDENT>query = queries.get(metric).format(inst)<EOL>futures_dict[inst] = get_metrics(cluster, environ, topology, timerange, query)<EOL><DEDENT>res = yield futures_dict<EOL>if not is_max:<EOL><INDENT>timelines = []<EOL>for key in res:<EOL><INDENT>result = res[key]<EOL>if len(result["<STR_LIT>"]) > <NUM_LIT:0>:<EOL><INDENT>result["<STR_LIT>"][<NUM_LIT:0>]["<STR_LIT>"] = key<EOL><DEDENT>timelines.extend(result["<STR_LIT>"])<EOL><DEDENT>result = self.get_metric_response(timerange, timelines, is_max)<EOL><DEDENT>else:<EOL><INDENT>data = self.compute_max(res.values())<EOL>result = self.get_metric_response(timerange, data, is_max)<EOL><DEDENT>raise tornado.gen.Return(result)<EOL>

:param cluster:
:param metric:
:param topology:
:param component:
:param instance:
:param timerange:
:param isMax:
:param environ:
:return:


if len(multi_ts) > <NUM_LIT:0> and len(multi_ts[<NUM_LIT:0>]["<STR_LIT>"]) > <NUM_LIT:0>:<EOL><INDENT>keys = multi_ts[<NUM_LIT:0>]["<STR_LIT>"][<NUM_LIT:0>]["<STR_LIT:data>"].keys()<EOL>timelines = ([res["<STR_LIT>"][<NUM_LIT:0>]["<STR_LIT:data>"][key] for key in keys] for res in multi_ts)<EOL>values = (max(v) for v in zip(*timelines))<EOL>return dict(zip(keys, values))<EOL><DEDENT>return {}<EOL>


def get_metric_response(self, timerange, data, isMax):

:param timerange:
:param data:
:param isMax:
:return:


def get_query(self, metric, component, instance):

q = queries.get(metric)<EOL>return q.format(component, instance)<EOL>

:param metric:
:param component:
:param instance:
:return:


def create_tar(tar_filename, files, config_dir, config_files):

with contextlib.closing(tarfile.open(tar_filename, '<STR_LIT>', dereference=True)) as tar:<EOL><INDENT>for filename in files:<EOL><INDENT>if os.path.isfile(filename):<EOL><INDENT>tar.add(filename, arcname=os.path.basename(filename))<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % filename)<EOL><DEDENT><DEDENT>if os.path.isdir(config_dir):<EOL><INDENT>tar.add(config_dir, arcname=get_heron_sandbox_conf_dir())<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % config_dir)<EOL><DEDENT>for filename in config_files:<EOL><INDENT>if os.path.isfile(filename):<EOL><INDENT>arcfile = os.path.join(get_heron_sandbox_conf_dir(), os.path.basename(filename))<EOL>tar.add(filename, arcname=arcfile)<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % filename)<EOL><DEDENT><DEDENT><DEDENT>

Create a tar file with a given set of files


<EOL>subparsers_actions = [action for action in parser._actions<EOL>if isinstance(action, argparse._SubParsersAction)]<EOL>for subparsers_action in subparsers_actions:<EOL><INDENT>for choice, subparser in subparsers_action.choices.items():<EOL><INDENT>if choice == command:<EOL><INDENT>return subparser<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

Retrieve the given subparser from parser


command = ['<STR_LIT>', '<STR_LIT>', x]<EOL>p = subprocess.Popen(command, stdout=subprocess.PIPE)<EOL>result = p.communicate()<EOL>output = result[<NUM_LIT:0>]<EOL>lines = output.split("<STR_LIT:\n>")<EOL>return lines[<NUM_LIT:0>]<EOL>


return x<EOL>


if sys.platform == '<STR_LIT>':<EOL><INDENT>return cygpath(x)<EOL><DEDENT>return identity(x)<EOL>


return '<STR_LIT::>'.join(map(normalized_class_path, jars))<EOL>

Get the normalized class path of all jars


go_above_dirs = <NUM_LIT:9><EOL>path = "<STR_LIT:/>".join(os.path.realpath(__file__).split('<STR_LIT:/>')[:-go_above_dirs])<EOL>return normalized_class_path(path)<EOL>

This will extract heron directory from .pex file.

For example,
when __file__ is '/Users/heron-user/bin/heron/heron/tools/common/src/python/utils/config.pyc', and
its real path is '/Users/heron-user/.heron/bin/heron/tools/common/src/python/utils/config.pyc',
the internal variable ``path`` would be '/Users/heron-user/.heron', which is the heron directory

This means the variable `go_above_dirs` below is 9.

:return: root location of the .pex file


go_above_dirs = <NUM_LIT:7><EOL>path = "<STR_LIT:/>".join(os.path.realpath(__file__).split('<STR_LIT:/>')[:-go_above_dirs])<EOL>return normalized_class_path(path)<EOL>

This will extract heron directory from .pex file,
with `zip_safe = False' Bazel flag added when building this .pex file

For example,
when __file__'s real path is
  '/Users/heron-user/.pex/code/xxxyyy/heron/tools/common/src/python/utils/config.pyc', and
the internal variable ``path`` would be '/Users/heron-user/.pex/code/xxxyyy/',
which is the root PEX directory

This means the variable `go_above_dirs` below is 7.

:return: root location of the .pex file.


bin_path = os.path.join(get_heron_dir(), BIN_DIR)<EOL>return bin_path<EOL>

This will provide heron bin directory from .pex file.
:return: absolute path of heron lib directory


conf_path = os.path.join(get_heron_dir(), CONF_DIR)<EOL>return conf_path<EOL>

This will provide heron conf directory from .pex file.
:return: absolute path of heron conf directory


lib_path = os.path.join(get_heron_dir(), LIB_DIR)<EOL>return lib_path<EOL>

This will provide heron lib directory from .pex file.
:return: absolute path of heron lib directory


return os.path.join(get_heron_dir(), RELEASE_YAML)<EOL>

This will provide the path to heron release.yaml file
:return: absolute path of heron release.yaml file in CLI


return os.path.join(get_zipped_heron_dir(), ZIPPED_RELEASE_YAML)<EOL>

This will provide the path to heron release.yaml file.
To be used for .pex file built with `zip_safe = False` flag.
For example, `heron-ui'.

:return: absolute path of heron release.yaml file


return os.path.join(default_config_path, cluster)<EOL>

This will provide heron cluster config directory, if config path is default
:return: absolute path of heron cluster conf directory


return SANDBOX_CONF_DIR<EOL>

This will provide heron conf directory in the sandbox
:return: relative path of heron sandbox conf directory


heron_lib_dir = get_heron_lib_dir()<EOL>heron_libs = [os.path.join(heron_lib_dir, f) for f in local_jars]<EOL>return heron_libs<EOL>

Get all the heron lib jars with the absolute paths


return cluster_role_env.split('<STR_LIT:/>')[<NUM_LIT:0>]<EOL>

Get the cluster to which topology is submitted


parts = cluster_role_env.split('<STR_LIT:/>')[:<NUM_LIT:3>]<EOL>if not os.path.isdir(config_path):<EOL><INDENT>Log.error("<STR_LIT>" % config_path)<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>if len(parts) < <NUM_LIT:3>:<EOL><INDENT>cli_conf_file = os.path.join(config_path, CLIENT_YAML)<EOL>if not os.path.isfile(cli_conf_file):<EOL><INDENT>if len(parts) == <NUM_LIT:1>:<EOL><INDENT>parts.append(getpass.getuser())<EOL><DEDENT>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>parts.append(ENVIRON)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>cli_confs = {}<EOL>with open(cli_conf_file, '<STR_LIT:r>') as conf_file:<EOL><INDENT>tmp_confs = yaml.load(conf_file)<EOL>if tmp_confs is not None:<EOL><INDENT>cli_confs = tmp_confs<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>" % (CLIENT_YAML))<EOL><DEDENT><DEDENT>if len(parts) == <NUM_LIT:1>:<EOL><INDENT>if (ROLE_REQUIRED in cli_confs) and (cli_confs[ROLE_REQUIRED] is True):<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (cluster_role_env, ROLE_REQUIRED, cli_conf_file))<EOL><DEDENT>else:<EOL><INDENT>parts.append(getpass.getuser())<EOL><DEDENT><DEDENT>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>if (ENV_REQUIRED in cli_confs) and (cli_confs[ENV_REQUIRED] is True):<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (cluster_role_env, ENV_REQUIRED, cli_conf_file))<EOL><DEDENT>else:<EOL><INDENT>parts.append(ENVIRON)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if len(parts[<NUM_LIT:0>]) == <NUM_LIT:0> or len(parts[<NUM_LIT:1>]) == <NUM_LIT:0> or len(parts[<NUM_LIT:2>]) == <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return (parts[<NUM_LIT:0>], parts[<NUM_LIT:1>], parts[<NUM_LIT:2>])<EOL>

Parse cluster/[role]/[environ], supply default, if not provided, not required


parts = cluster_role_env.split('<STR_LIT:/>')[:<NUM_LIT:3>]<EOL>if len(parts) == <NUM_LIT:3>:<EOL><INDENT>return (parts[<NUM_LIT:0>], parts[<NUM_LIT:1>], parts[<NUM_LIT:2>])<EOL><DEDENT>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>return (parts[<NUM_LIT:0>], parts[<NUM_LIT:1>], "<STR_LIT>")<EOL><DEDENT>if len(parts) == <NUM_LIT:1>:<EOL><INDENT>return (parts[<NUM_LIT:0>], "<STR_LIT>", "<STR_LIT>")<EOL><DEDENT>return ("<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>

Parse cluster/[role]/[environ], supply empty string, if not provided


<EOL>cli_conf_file = os.path.join(config_path, CLIENT_YAML)<EOL>if not os.path.isfile(cli_conf_file):<EOL><INDENT>return True<EOL><DEDENT>client_confs = {}<EOL>with open(cli_conf_file, '<STR_LIT:r>') as conf_file:<EOL><INDENT>client_confs = yaml.load(conf_file)<EOL>if not client_confs:<EOL><INDENT>return True<EOL><DEDENT>role_present = True if len(cluster_role_env[<NUM_LIT:1>]) > <NUM_LIT:0> else False<EOL>if ROLE_REQUIRED in client_confs and client_confs[ROLE_REQUIRED] and not role_present:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (cluster_role_env, ROLE_REQUIRED, cli_conf_file))<EOL><DEDENT>environ_present = True if len(cluster_role_env[<NUM_LIT:2>]) > <NUM_LIT:0> else False<EOL>if ENV_REQUIRED in client_confs and client_confs[ENV_REQUIRED] and not environ_present:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% (cluster_role_env, ENV_REQUIRED, cli_conf_file))<EOL><DEDENT><DEDENT>return True<EOL>

Check cluster/[role]/[environ], if they are required


cmap = config_map[cluster_role_env[<NUM_LIT:0>]]<EOL>role_present = True if len(cluster_role_env[<NUM_LIT:1>]) > <NUM_LIT:0> else False<EOL>if ROLE_KEY in cmap and cmap[ROLE_KEY] and not role_present:<EOL><INDENT>raise Exception("<STR_LIT>"% (cluster_role_env))<EOL><DEDENT>environ_present = True if len(cluster_role_env[<NUM_LIT:2>]) > <NUM_LIT:0> else False<EOL>if ENVIRON_KEY in cmap and cmap[ENVIRON_KEY] and not environ_present:<EOL><INDENT>raise Exception("<STR_LIT>"% (cluster_role_env))<EOL><DEDENT>return True<EOL>


if len(cluster_role_env[<NUM_LIT:1>]) == <NUM_LIT:0> and len(cluster_role_env[<NUM_LIT:2>]) == <NUM_LIT:0>:<EOL><INDENT>return (cluster_role_env[<NUM_LIT:0>], getpass.getuser(), ENVIRON)<EOL><DEDENT>return (cluster_role_env[<NUM_LIT:0>], cluster_role_env[<NUM_LIT:1>], cluster_role_env[<NUM_LIT:2>])<EOL>

if role is not provided, supply userid
if environ is not provided, supply 'default'


overrides = parse_override_config(namespace)<EOL>try:<EOL><INDENT>tmp_dir = tempfile.mkdtemp()<EOL>override_config_file = os.path.join(tmp_dir, OVERRIDE_YAML)<EOL>with open(override_config_file, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(yaml.dump(overrides))<EOL><DEDENT>return override_config_file<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise Exception("<STR_LIT>" % str(e))<EOL><DEDENT>

Parse the command line for overriding the defaults and
create an override file.


overrides = dict()<EOL>for config in namespace:<EOL><INDENT>kv = config.split("<STR_LIT:=>")<EOL>if len(kv) != <NUM_LIT:2>:<EOL><INDENT>raise Exception("<STR_LIT>" % config)<EOL><DEDENT>if kv[<NUM_LIT:1>] in ['<STR_LIT:true>', '<STR_LIT:True>', '<STR_LIT>']:<EOL><INDENT>overrides[kv[<NUM_LIT:0>]] = True<EOL><DEDENT>elif kv[<NUM_LIT:1>] in ['<STR_LIT:false>', '<STR_LIT:False>', '<STR_LIT>']:<EOL><INDENT>overrides[kv[<NUM_LIT:0>]] = False<EOL><DEDENT>else:<EOL><INDENT>overrides[kv[<NUM_LIT:0>]] = kv[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>return overrides<EOL>

Parse the command line for overriding the defaults


java_home = os.environ.get("<STR_LIT>")<EOL>return os.path.join(java_home, BIN_DIR, "<STR_LIT>")<EOL>


<EOL>if "<STR_LIT>" not in os.environ:<EOL><INDENT>Log.error("<STR_LIT>")<EOL>return False<EOL><DEDENT>java_path = get_java_path()<EOL>if os.path.isfile(java_path) and os.access(java_path, os.X_OK):<EOL><INDENT>return True<EOL><DEDENT>Log.error("<STR_LIT>")<EOL>return False<EOL>


release_file = get_heron_release_file()<EOL>if not os.path.isfile(release_file):<EOL><INDENT>Log.error("<STR_LIT>" % release_file)<EOL>return False<EOL><DEDENT>return True<EOL>


if zipped_pex:<EOL><INDENT>release_file = get_zipped_heron_release_file()<EOL><DEDENT>else:<EOL><INDENT>release_file = get_heron_release_file()<EOL><DEDENT>with open(release_file) as release_info:<EOL><INDENT>release_map = yaml.load(release_info)<EOL>release_items = sorted(release_map.items(), key=lambda tup: tup[<NUM_LIT:0>])<EOL>for key, value in release_items:<EOL><INDENT>print("<STR_LIT>" % (key, value))<EOL><DEDENT><DEDENT>

Print build_info from release.yaml

    :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'.


if zipped_pex:<EOL><INDENT>release_file = get_zipped_heron_release_file()<EOL><DEDENT>else:<EOL><INDENT>release_file = get_heron_release_file()<EOL><DEDENT>with open(release_file) as release_info:<EOL><INDENT>for line in release_info:<EOL><INDENT>trunks = line[:-<NUM_LIT:1>].split('<STR_LIT:U+0020>')<EOL>if trunks[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>return trunks[-<NUM_LIT:1>].replace("<STR_LIT:'>", "<STR_LIT>")<EOL><DEDENT><DEDENT>return '<STR_LIT>'<EOL><DEDENT>

Print version from release.yaml

    :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'.


index = <NUM_LIT:0><EOL>found = False<EOL>for lelem in command_args:<EOL><INDENT>if lelem == '<STR_LIT>' and not found:<EOL><INDENT>break<EOL><DEDENT>if lelem == param:<EOL><INDENT>found = True<EOL>break<EOL><DEDENT>index = index + <NUM_LIT:1><EOL><DEDENT>if found:<EOL><INDENT>command_args.insert(index + <NUM_LIT:1>, '<STR_LIT:True>')<EOL><DEDENT>return command_args<EOL>

:param param:
:param command_args:
:return:


args1 = insert_bool('<STR_LIT>', command_line_args)<EOL>args2 = insert_bool('<STR_LIT>', args1)<EOL>return args2<EOL>


<EOL>if path.endswith('<STR_LIT:*>'):<EOL><INDENT>Log.debug('<STR_LIT>', path[:-<NUM_LIT:1>])<EOL>if os.path.isdir(path[:-<NUM_LIT:1>]):<EOL><INDENT>return True<EOL><DEDENT>return False<EOL><DEDENT>Log.debug('<STR_LIT>', path)<EOL>if os.path.isdir(path):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>Log.debug('<STR_LIT>', path)<EOL>if os.path.isfile(path):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Check if an entry in the class path exists as either a directory or a file


paths = classpath.split('<STR_LIT::>')<EOL>for path_entry in paths:<EOL><INDENT>if not valid_path(path_entry.strip()):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Given a java classpath, check whether the path entries are valid or not


self.locations = state_locations<EOL>self.validate_state_locations()<EOL>


names = map(lambda loc: loc["<STR_LIT:name>"], self.locations)<EOL>assert len(names) == len(set(names)), "<STR_LIT>"<EOL>

Names of all state locations must be unique.


return self.locations<EOL>


return filter(lambda location: location['<STR_LIT:type>'] == location_type, self.locations)<EOL>

Return a list of state locations of a particular type.


self.monitoring_thread_stop_signal = False<EOL>self.monitoring_thread.start()<EOL>


self.monitoring_thread_stop_signal = True<EOL>self.event.set()<EOL>


def trigger_watches_based_on_files(watchers, path, directory, ProtoClass):<EOL><INDENT>"""<STR_LIT>"""<EOL>for topology, callbacks in watchers.items():<EOL><INDENT>file_path = os.path.join(path, topology)<EOL>data = "<STR_LIT>"<EOL>if os.path.exists(file_path):<EOL><INDENT>with open(os.path.join(path, topology)) as f:<EOL><INDENT>data = f.read()<EOL><DEDENT><DEDENT>if topology not in directory or data != directory[topology]:<EOL><INDENT>proto_object = ProtoClass()<EOL>proto_object.ParseFromString(data)<EOL>for callback in callbacks:<EOL><INDENT>callback(proto_object)<EOL><DEDENT>directory[topology] = data<EOL><DEDENT><DEDENT><DEDENT>while not self.monitoring_thread_stop_signal:<EOL><INDENT>topologies_path = self.get_topologies_path()<EOL>topologies = []<EOL>if os.path.isdir(topologies_path):<EOL><INDENT>topologies = list(filter(<EOL>lambda f: os.path.isfile(os.path.join(topologies_path, f)),<EOL>os.listdir(topologies_path)))<EOL><DEDENT>if set(topologies) != set(self.topologies_directory):<EOL><INDENT>for callback in self.topologies_watchers:<EOL><INDENT>callback(topologies)<EOL><DEDENT><DEDENT>self.topologies_directory = topologies<EOL>trigger_watches_based_on_files(<EOL>self.topology_watchers, topologies_path, self.topologies_directory, Topology)<EOL>execution_state_path = os.path.dirname(self.get_execution_state_path("<STR_LIT>"))<EOL>trigger_watches_based_on_files(<EOL>self.execution_state_watchers, execution_state_path,<EOL>self.execution_state_directory, ExecutionState)<EOL>packing_plan_path = os.path.dirname(self.get_packing_plan_path("<STR_LIT>"))<EOL>trigger_watches_based_on_files(<EOL>self.packing_plan_watchers, packing_plan_path, self.packing_plan_directory, PackingPlan)<EOL>pplan_path = os.path.dirname(self.get_pplan_path("<STR_LIT>"))<EOL>trigger_watches_based_on_files(<EOL>self.pplan_watchers, pplan_path,<EOL>self.pplan_directory, PhysicalPlan)<EOL>tmaster_path = os.path.dirname(self.get_tmaster_path("<STR_LIT>"))<EOL>trigger_watches_based_on_files(<EOL>self.tmaster_watchers, tmaster_path,<EOL>self.tmaster_directory, TMasterLocation)<EOL>scheduler_location_path = os.path.dirname(self.get_scheduler_location_path("<STR_LIT>"))<EOL>trigger_watches_based_on_files(<EOL>self.scheduler_location_watchers, scheduler_location_path,<EOL>self.scheduler_location_directory, SchedulerLocation)<EOL>self.event.wait(timeout=<NUM_LIT:5>)<EOL><DEDENT>

Monitor the rootpath and call the callback
corresponding to the change.
This monitoring happens periodically. This function
is called in a seperate thread from the main thread,
because it sleeps for the intervals between each poll.


if callback:<EOL><INDENT>self.topologies_watchers.append(callback)<EOL><DEDENT>else:<EOL><INDENT>topologies_path = self.get_topologies_path()<EOL>return filter(lambda f: os.path.isfile(os.path.join(topologies_path, f)),<EOL>os.listdir(topologies_path))<EOL><DEDENT>


def get_topology(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.topology_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>topology_path = self.get_topology_path(topologyName)<EOL>with open(topology_path) as f:<EOL><INDENT>data = f.read()<EOL>topology = Topology()<EOL>topology.ParseFromString(data)<EOL>return topology<EOL><DEDENT><DEDENT>


def create_topology(self, topologyName, topology):

pass<EOL>

Create path is currently not supported in file based state manager.


pass<EOL>

Delete path is currently not supported in file based state manager.


def get_packing_plan(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.packing_plan_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>packing_plan_path = self.get_packing_plan_path(topologyName)<EOL>with open(packing_plan_path) as f:<EOL><INDENT>data = f.read()<EOL>packing_plan = PackingPlan()<EOL>packing_plan.ParseFromString(data)<EOL><DEDENT><DEDENT>


def get_pplan(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.pplan_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>pplan_path = self.get_pplan_path(topologyName)<EOL>with open(pplan_path) as f:<EOL><INDENT>data = f.read()<EOL>pplan = PhysicalPlan()<EOL>pplan.ParseFromString(data)<EOL>return pplan<EOL><DEDENT><DEDENT>


def create_pplan(self, topologyName, pplan):

pass<EOL>


pass<EOL>


def get_execution_state(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.execution_state_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>execution_state_path = self.get_execution_state_path(topologyName)<EOL>with open(execution_state_path) as f:<EOL><INDENT>data = f.read()<EOL>executionState = ExecutionState()<EOL>executionState.ParseFromString(data)<EOL>return executionState<EOL><DEDENT><DEDENT>


def create_execution_state(self, topologyName, executionState):

pass<EOL>


def get_tmaster(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.tmaster_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>tmaster_path = self.get_tmaster_path(topologyName)<EOL>with open(tmaster_path) as f:<EOL><INDENT>data = f.read()<EOL>tmaster = TMasterLocation()<EOL>tmaster.ParseFromString(data)<EOL>return tmaster<EOL><DEDENT><DEDENT>


def get_scheduler_location(self, topologyName, callback=None):

if callback:<EOL><INDENT>self.scheduler_location_watchers[topologyName].append(callback)<EOL><DEDENT>else:<EOL><INDENT>scheduler_location_path = self.get_scheduler_location_path(topologyName)<EOL>with open(scheduler_location_path) as f:<EOL><INDENT>data = f.read()<EOL>scheduler_location = SchedulerLocation()<EOL>scheduler_location.ParseFromString(data)<EOL>return scheduler_location<EOL><DEDENT><DEDENT>


log_format = "<STR_LIT>"<EOL>date_format = '<STR_LIT>'<EOL>logging.basicConfig(format=log_format, datefmt=date_format)<EOL>Log.setLevel(level)<EOL>if logfile is not None:<EOL><INDENT>fh = logging.FileHandler(logfile)<EOL>fh.setFormatter(logging.Formatter(log_format))<EOL>Log.addHandler(fh)<EOL><DEDENT>


state_managers = []<EOL>try:<EOL><INDENT>state_managers.extend(get_all_zk_state_managers(conf))<EOL>state_managers.extend(get_all_file_state_managers(conf))<EOL>return state_managers<EOL><DEDENT>except Exception as ex:<EOL><INDENT>LOG.error("<STR_LIT>")<EOL>raise ex<EOL><DEDENT>

@param conf - An instance of Config class
Reads the config for requested state managers.
Instantiates them, start and then return them.


state_managers = []<EOL>state_locations = conf.get_state_locations_of_type("<STR_LIT>")<EOL>for location in state_locations:<EOL><INDENT>name = location['<STR_LIT:name>']<EOL>hostport = location['<STR_LIT>']<EOL>hostportlist = []<EOL>for hostportpair in hostport.split('<STR_LIT:U+002C>'):<EOL><INDENT>host = None<EOL>port = None<EOL>if '<STR_LIT::>' in hostport:<EOL><INDENT>hostandport = hostportpair.split('<STR_LIT::>')<EOL>if len(hostandport) == <NUM_LIT:2>:<EOL><INDENT>host = hostandport[<NUM_LIT:0>]<EOL>port = int(hostandport[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>if not host or not port:<EOL><INDENT>raise Exception("<STR_LIT>" % (name))<EOL><DEDENT>hostportlist.append((host, port))<EOL><DEDENT>tunnelhost = location['<STR_LIT>']<EOL>rootpath = location['<STR_LIT>']<EOL>LOG.info("<STR_LIT>" + str(hostportlist) + "<STR_LIT>" + rootpath)<EOL>state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost)<EOL>state_managers.append(state_manager)<EOL><DEDENT>return state_managers<EOL>

Creates all the zookeeper state_managers and returns
them in a list


state_managers = []<EOL>state_locations = conf.get_state_locations_of_type("<STR_LIT:file>")<EOL>for location in state_locations:<EOL><INDENT>name = location['<STR_LIT:name>']<EOL>rootpath = os.path.expanduser(location['<STR_LIT>'])<EOL>LOG.info("<STR_LIT>" + rootpath)<EOL>state_manager = FileStateManager(name, rootpath)<EOL>state_managers.append(state_manager)<EOL><DEDENT>return state_managers<EOL>


return KazooClient(hostportlist)<EOL>

For Unit testing, replace this method to not
Actually return a client


if self.is_host_port_reachable():<EOL><INDENT>self.client = self._kazoo_client(_makehostportlist(self.hostportlist))<EOL><DEDENT>else:<EOL><INDENT>localhostports = self.establish_ssh_tunnel()<EOL>self.client = self._kazoo_client(_makehostportlist(localhostports))<EOL><DEDENT>self.client.start()<EOL>def on_connection_change(state):<EOL><INDENT>"""<STR_LIT>"""<EOL>LOG.info("<STR_LIT>" + state)<EOL><DEDENT>self.client.add_listener(on_connection_change)<EOL>


self.client.stop()<EOL>self.terminate_ssh_tunnel()<EOL>


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>try:<EOL><INDENT>self.client.ensure_path(self.get_topologies_path())<EOL>self._get_topologies_with_watch(callback, isWatching)<EOL><DEDENT>except NoNodeError:<EOL><INDENT>self.client.stop()<EOL>path = self.get_topologies_path()<EOL>raise_(StateException("<STR_LIT>" % (path),<EOL>StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>return ret["<STR_LIT:result>"]<EOL>


def _get_topologies_with_watch(self, callback, isWatching):

path = self.get_topologies_path()<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.ChildrenWatch(path)<EOL>def watch_topologies(topologies):<EOL><INDENT>"""<STR_LIT>"""<EOL>callback(topologies)<EOL>return isWatching<EOL><DEDENT>

Helper function to get topologies with
a callback. The future watch is placed
only if isWatching is True.


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_topology_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def _get_topology_with_watch(self, topologyName, callback, isWatching):

path = self.get_topology_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_topology(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>topology = Topology()<EOL>topology.ParseFromString(data)<EOL>callback(topology)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>

Helper function to get pplan with
a callback. The future watch is placed
only if isWatching is True.


if not topology or not topology.IsInitialized():<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>path = self.get_topology_path(topologyName)<EOL>LOG.info("<STR_LIT>".format(<EOL>topologyName, path))<EOL>topologyString = topology.SerializeToString()<EOL>try:<EOL><INDENT>self.client.create(path, value=topologyString, makepath=True)<EOL>return True<EOL><DEDENT>except NoNodeError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except NodeExistsError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except ZookeeperError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except Exception:<EOL><INDENT>raise<EOL><DEDENT>


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_packing_plan_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def _get_packing_plan_with_watch(self, topologyName, callback, isWatching):

path = self.get_packing_plan_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_packing_plan(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>packing_plan = PackingPlan()<EOL>packing_plan.ParseFromString(data)<EOL>callback(packing_plan)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>

Helper function to get packing_plan with
a callback. The future watch is placed
only if isWatching is True.


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_pplan_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def _get_pplan_with_watch(self, topologyName, callback, isWatching):

path = self.get_pplan_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_pplan(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>pplan = PhysicalPlan()<EOL>pplan.ParseFromString(data)<EOL>callback(pplan)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>


if not pplan or not pplan.IsInitialized():<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>path = self.get_pplan_path(topologyName)<EOL>LOG.info("<STR_LIT>".format(<EOL>topologyName, path))<EOL>pplanString = pplan.SerializeToString()<EOL>try:<EOL><INDENT>self.client.create(path, value=pplanString, makepath=True)<EOL>return True<EOL><DEDENT>except NoNodeError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except NodeExistsError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except ZookeeperError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except Exception:<EOL><INDENT>raise<EOL><DEDENT>


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_execution_state_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def _get_execution_state_with_watch(self, topologyName, callback, isWatching):

path = self.get_execution_state_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_execution_state(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>executionState = ExecutionState()<EOL>executionState.ParseFromString(data)<EOL>callback(executionState)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>

Helper function to get execution state with
a callback. The future watch is placed
only if isWatching is True.


if not executionState or not executionState.IsInitialized():<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_PROTOBUF_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>path = self.get_execution_state_path(topologyName)<EOL>LOG.info("<STR_LIT>".format(<EOL>topologyName, path))<EOL>executionStateString = executionState.SerializeToString()<EOL>try:<EOL><INDENT>self.client.create(path, value=executionStateString, makepath=True)<EOL>return True<EOL><DEDENT>except NoNodeError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except NodeExistsError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_NODE_EXISTS_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except ZookeeperError:<EOL><INDENT>raise_(StateException("<STR_LIT>",<EOL>StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>except Exception:<EOL><INDENT>raise<EOL><DEDENT>


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_tmaster_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def _get_tmaster_with_watch(self, topologyName, callback, isWatching):

path = self.get_tmaster_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_tmaster(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>tmaster = TMasterLocation()<EOL>tmaster.ParseFromString(data)<EOL>callback(tmaster)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>


isWatching = False<EOL>ret = {<EOL>"<STR_LIT:result>": None<EOL>}<EOL>if callback:<EOL><INDENT>isWatching = True<EOL><DEDENT>else:<EOL><INDENT>def callback(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret["<STR_LIT:result>"] = data<EOL><DEDENT><DEDENT>self._get_scheduler_location_with_watch(topologyName, callback, isWatching)<EOL>return ret["<STR_LIT:result>"]<EOL>


def _get_scheduler_location_with_watch(self, topologyName, callback, isWatching):

path = self.get_scheduler_location_path(topologyName)<EOL>if isWatching:<EOL><INDENT>LOG.info("<STR_LIT>" + path)<EOL><DEDENT>@self.client.DataWatch(path)<EOL>def watch_scheduler_location(data, stats):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data:<EOL><INDENT>scheduler_location = SchedulerLocation()<EOL>scheduler_location.ParseFromString(data)<EOL>callback(scheduler_location)<EOL><DEDENT>else:<EOL><INDENT>callback(None)<EOL><DEDENT>return isWatching<EOL><DEDENT>

Helper function to get scheduler location with
a callback. The future watch is placed
only if isWatching is True.


with open(state_manager_config_file, '<STR_LIT:r>') as stream:<EOL><INDENT>config = yaml.load(stream)<EOL><DEDENT>home_dir = os.path.expanduser("<STR_LIT>")<EOL>wildcards = {<EOL>"<STR_LIT>" : home_dir,<EOL>"<STR_LIT>" : home_dir,<EOL>"<STR_LIT>" : cluster,<EOL>}<EOL>if os.getenv('<STR_LIT>'):<EOL><INDENT>wildcards["<STR_LIT>"] = os.getenv('<STR_LIT>')<EOL><DEDENT>config = __replace(config, wildcards, state_manager_config_file)<EOL>if overrides:<EOL><INDENT>config.update(overrides)<EOL><DEDENT>state_manager_location = {<EOL>'<STR_LIT:type>': '<STR_LIT:file>',<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT:127.0.0.1>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>}<EOL>key_mappings = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>}<EOL>for config_key in key_mappings:<EOL><INDENT>if config_key in config:<EOL><INDENT>state_manager_location[key_mappings[config_key]] = config[config_key]<EOL><DEDENT><DEDENT>state_manager_class = config['<STR_LIT>']<EOL>if state_manager_class == '<STR_LIT>':<EOL><INDENT>state_manager_location['<STR_LIT:type>'] = '<STR_LIT>'<EOL>state_manager_location['<STR_LIT:name>'] = '<STR_LIT>'<EOL><DEDENT>return [state_manager_location]<EOL>

Reads configs to determine which state manager to use and converts them to state manager
    locations. Handles a subset of config wildcard substitution supported in the substitute method in
    org.apache.heron.spi.common.Misc.java


def __replace(config, wildcards, config_file):

for config_key in config:<EOL><INDENT>config_value = config[config_key]<EOL>original_value = config_value<EOL>if isinstance(config_value, str):<EOL><INDENT>for token in wildcards:<EOL><INDENT>if wildcards[token]:<EOL><INDENT>config_value = config_value.replace(token, wildcards[token])<EOL><DEDENT><DEDENT>found = re.findall(r'<STR_LIT>', config_value)<EOL>if found:<EOL><INDENT>raise ValueError("<STR_LIT>" %<EOL>(config_key, original_value, config_file, "<STR_LIT:U+002CU+0020>".join(found)))<EOL><DEDENT>config[config_key] = config_value<EOL><DEDENT><DEDENT>return config<EOL>

For each kvp in config, do wildcard substitution on the values


return self.__hostport<EOL>

Getter for the path where the heron states are stored.


self.__hostport = newRootPath<EOL>

Setter for the path where the heron states are stored.


return self.__tunnelhost<EOL>

Getter for the tunnelhost to create the tunnel if host is not accessible


self.__tunnelhost = newTunnelHost<EOL>

Setter for the tunnelhost to create the tunnel if host is not accessible


for hostport in self.hostportlist:<EOL><INDENT>try:<EOL><INDENT>socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS)<EOL>return True<EOL><DEDENT>except:<EOL><INDENT>LOG.info("<STR_LIT>"<EOL>% (self.name, hostport[<NUM_LIT:0>], hostport[<NUM_LIT:1>]))<EOL>continue<EOL><DEDENT><DEDENT>return False<EOL>

Returns true if the host is reachable. In some cases, it may not be reachable a tunnel
must be used.


s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)<EOL>s.bind(('<STR_LIT:127.0.0.1>', <NUM_LIT:0>))<EOL>_, port = s.getsockname()<EOL>s.close()<EOL>return port<EOL>

Pick an unused port. There is a slight chance that this wont work.


localportlist = []<EOL>for (host, port) in self.hostportlist:<EOL><INDENT>localport = self.pick_unused_port()<EOL>self.tunnel.append(subprocess.Popen(<EOL>('<STR_LIT>', self.tunnelhost, '<STR_LIT>' % (localport, host, port))))<EOL>localportlist.append(('<STR_LIT:127.0.0.1>', localport))<EOL><DEDENT>return localportlist<EOL>

Establish an ssh tunnel for each local host and port
that can be used to communicate with the state host.


pass<EOL>

If the state manager needs to connect to a remote host.


pass<EOL>

If the state manager had connected to a remote server, it would need to stop as well.


@abc.abstractmethod<EOL><INDENT>def get_packing_plan(self, topologyName, callback=None):<DEDENT>

pass<EOL>

Gets the packing_plan for the topology.
If the callback is provided,
sets watch on the path and calls the callback
with the new packing_plan.


self.delete_pplan(topologyName)<EOL>self.delete_execution_state(topologyName)<EOL>self.delete_topology(topologyName)<EOL>

Removes the topology entry from:
1. topologies list,
2. pplan,
3. execution_state, and


def status_finish(ret):<EOL><INDENT>self.set_status(ret)<EOL>self.finish()<EOL><DEDENT>def kill_parent():<EOL><INDENT>status_finish(<NUM_LIT:200>)<EOL>logger.info("<STR_LIT>")<EOL>os.killpg(os.getppid(), signal.SIGTERM)<EOL><DEDENT>logger = logging.getLogger(__file__)<EOL>logger.info("<STR_LIT>")<EOL>data = dict(urlparse.parse_qsl(self.request.body))<EOL>sharedSecret = data.get('<STR_LIT>')<EOL>if sharedSecret != options.secret:<EOL><INDENT>status_finish(<NUM_LIT>)<EOL>return<EOL><DEDENT>instanceId = data.get('<STR_LIT>')<EOL>if instanceId:<EOL><INDENT>filepath = instanceId + '<STR_LIT>'<EOL>if os.path.isfile(filepath): <EOL><INDENT>if instanceId.startswith('<STR_LIT>'): <EOL><INDENT>kill_parent()<EOL><DEDENT>else: <EOL><INDENT>fh = open(filepath)<EOL>firstLine = int(fh.readline())<EOL>fh.close()<EOL>logger.info("<STR_LIT>" + instanceId + "<STR_LIT:U+0020>" + str(firstLine))<EOL>os.kill(firstLine, signal.SIGTERM)<EOL>status_finish(<NUM_LIT:200>)<EOL><DEDENT><DEDENT>else: <EOL><INDENT>logger.info(filepath + "<STR_LIT>")<EOL>status_finish(<NUM_LIT>)<EOL><DEDENT><DEDENT>else: <EOL><INDENT>kill_parent()<EOL><DEDENT>


if path is None:<EOL><INDENT>return {}<EOL><DEDENT>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>offset = self.get_argument("<STR_LIT>", default=-<NUM_LIT:1>)<EOL>length = self.get_argument("<STR_LIT>", default=-<NUM_LIT:1>)<EOL>if not os.path.isfile(path):<EOL><INDENT>return {}<EOL><DEDENT>data = utils.read_chunk(path, offset=offset, length=length, escape_data=True)<EOL>self.write(json.dumps(data))<EOL>self.finish()<EOL>


<EOL>self.connection_closed = True<EOL>


logging.debug("<STR_LIT>", path)<EOL>self.connection_closed = False<EOL>self.set_header("<STR_LIT>", "<STR_LIT>")<EOL>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>if path is None or not os.path.isfile(path):<EOL><INDENT>self.write("<STR_LIT>" % path)<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>length = int(<NUM_LIT:4> * <NUM_LIT> * <NUM_LIT>)<EOL>offset = int(<NUM_LIT:0>)<EOL>while True:<EOL><INDENT>data = utils.read_chunk(path, offset=offset, length=length, escape_data=False)<EOL>if self.connection_closed or '<STR_LIT:data>' not in data or len(data['<STR_LIT:data>']) < length:<EOL><INDENT>break<EOL><DEDENT>offset += length<EOL>self.write(data['<STR_LIT:data>'])<EOL>self.flush()<EOL><DEDENT>if '<STR_LIT:data>' in data:<EOL><INDENT>self.write(data['<STR_LIT:data>'])<EOL><DEDENT>self.finish()<EOL>


path = tornado.escape.url_unescape(path)<EOL>if not path:<EOL><INDENT>path = "<STR_LIT:.>"<EOL><DEDENT>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>listing = utils.get_listing(path)<EOL>file_stats = {}<EOL>for fn in listing:<EOL><INDENT>try:<EOL><INDENT>is_dir = False<EOL>formatted_stat = utils.format_prefix(fn, utils.get_stat(path, fn))<EOL>if stat.S_ISDIR(utils.get_stat(path, fn).st_mode):<EOL><INDENT>is_dir = True<EOL><DEDENT>file_stats[fn] = {<EOL>"<STR_LIT>": formatted_stat,<EOL>"<STR_LIT>": is_dir,<EOL>"<STR_LIT:path>": tornado.escape.url_escape(os.path.join(path, fn)),<EOL>}<EOL>if fn == "<STR_LIT:..>":<EOL><INDENT>path_fragments = path.split("<STR_LIT:/>")<EOL>if not path_fragments:<EOL><INDENT>file_stats[fn]["<STR_LIT:path>"] = "<STR_LIT:.>"<EOL><DEDENT>else:<EOL><INDENT>file_stats[fn]["<STR_LIT:path>"] = tornado.escape.url_escape("<STR_LIT:/>".join(path_fragments[:-<NUM_LIT:1>]))<EOL><DEDENT><DEDENT><DEDENT>except:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>self.write(json.dumps(file_stats))<EOL>self.finish()<EOL>


body = utils.str_cmd(['<STR_LIT>', pid], None, None)<EOL>self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(body))<EOL>self.finish()<EOL>


self.content_type = '<STR_LIT>'<EOL>self.write('<STR_LIT>')<EOL>self.finish()<EOL>


t = Template(utils.get_asset("<STR_LIT>"))<EOL>if path is None:<EOL><INDENT>self.set_status(<NUM_LIT>)<EOL>self.write("<STR_LIT>")<EOL>self.finish()<EOL>return<EOL><DEDENT>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>args = dict(<EOL>filename=path,<EOL>jquery=utils.get_asset("<STR_LIT>"),<EOL>pailer=utils.get_asset("<STR_LIT>"),<EOL>css=utils.get_asset("<STR_LIT>"),<EOL>)<EOL>self.write(t.generate(**args))<EOL>self.finish()<EOL>


body = utils.str_cmd(['<STR_LIT>', '<STR_LIT>', pid], None, None)<EOL>self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(body))<EOL>self.finish()<EOL>


if not path:<EOL><INDENT>path = "<STR_LIT:.>"<EOL><DEDENT>if not utils.check_path(path):<EOL><INDENT>self.write("<STR_LIT>")<EOL>self.set_status(<NUM_LIT>)<EOL>self.finish()<EOL>return<EOL><DEDENT>t = Template(utils.get_asset("<STR_LIT>"))<EOL>args = dict(<EOL>path=path,<EOL>listing=utils.get_listing(path),<EOL>format_prefix=utils.format_prefix,<EOL>stat=stat,<EOL>get_stat=utils.get_stat,<EOL>os=os,<EOL>css=utils.get_asset("<STR_LIT>")<EOL>)<EOL>self.write(t.generate(**args))<EOL>self.finish()<EOL>


utils.str_cmd(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'], None, None)<EOL>body = utils.str_cmd(['<STR_LIT>', '<STR_LIT>',<EOL>str(pid)], None, None)<EOL>utils.str_cmd(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'], None, None)<EOL>self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(body))<EOL>self.finish()<EOL>


self.content_type = '<STR_LIT:application/json>'<EOL>self.write(json.dumps(utils.chain([<EOL>['<STR_LIT>', '<STR_LIT>'],<EOL>['<STR_LIT>', instance_id],<EOL>['<STR_LIT>', '<STR_LIT>'],<EOL>['<STR_LIT>', '<STR_LIT>']])).strip())<EOL>self.finish()<EOL>


mode = sres.st_mode<EOL>root = (mode & <NUM_LIT>) >> <NUM_LIT:6><EOL>group = (mode & <NUM_LIT>) >> <NUM_LIT:3><EOL>user = (mode & <NUM_LIT>)<EOL>def stat_type(md):<EOL><INDENT>'''<STR_LIT>'''<EOL>if stat.S_ISDIR(md):<EOL><INDENT>return '<STR_LIT:d>'<EOL><DEDENT>elif stat.S_ISSOCK(md):<EOL><INDENT>return '<STR_LIT:s>'<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:->'<EOL><DEDENT><DEDENT>def triple(md):<EOL><INDENT>'''<STR_LIT>'''<EOL>return '<STR_LIT>' % (<EOL>'<STR_LIT:r>' if md & <NUM_LIT> else '<STR_LIT:->',<EOL>'<STR_LIT:w>' if md & <NUM_LIT> else '<STR_LIT:->',<EOL>'<STR_LIT:x>' if md & <NUM_LIT> else '<STR_LIT:->')<EOL><DEDENT>return '<STR_LIT>'.join([stat_type(mode), triple(root), triple(group), triple(user)])<EOL>

Format a line in the directory list based on the file's type and other attributes.


now = datetime.now()<EOL>dt = datetime.fromtimestamp(mtime)<EOL>return '<STR_LIT>' % (<EOL>dt.strftime('<STR_LIT>'), dt.day,<EOL>dt.year if dt.year != now.year else dt.strftime('<STR_LIT>'))<EOL>

Format the date associated with a file to be displayed in directory listing.


try:<EOL><INDENT>pwent = pwd.getpwuid(sres.st_uid)<EOL>user = pwent.pw_name<EOL><DEDENT>except KeyError:<EOL><INDENT>user = sres.st_uid<EOL><DEDENT>try:<EOL><INDENT>grent = grp.getgrgid(sres.st_gid)<EOL>group = grent.gr_name<EOL><DEDENT>except KeyError:<EOL><INDENT>group = sres.st_gid<EOL><DEDENT>return '<STR_LIT>' % (<EOL>format_mode(sres),<EOL>sres.st_nlink,<EOL>user,<EOL>group,<EOL>sres.st_size,<EOL>format_mtime(sres.st_mtime),<EOL>)<EOL>

Prefix to a filename in the directory listing. This is to make the
listing similar to an output of "ls -alh".


if path != "<STR_LIT:.>":<EOL><INDENT>listing = sorted(['<STR_LIT:..>'] + os.listdir(path))<EOL><DEDENT>else:<EOL><INDENT>listing = sorted(os.listdir(path))<EOL><DEDENT>return listing<EOL>

Returns the list of files and directories in a path.
Prepents a ".." (parent directory link) if path is not current dir.


return os.stat(os.path.join(path, filename))<EOL>


def read_chunk(filename, offset=-<NUM_LIT:1>, length=-<NUM_LIT:1>, escape_data=False):

try:<EOL><INDENT>length = int(length)<EOL>offset = int(offset)<EOL><DEDENT>except ValueError:<EOL><INDENT>return {}<EOL><DEDENT>if not os.path.isfile(filename):<EOL><INDENT>return {}<EOL><DEDENT>try:<EOL><INDENT>fstat = os.stat(filename)<EOL><DEDENT>except Exception:<EOL><INDENT>return {}<EOL><DEDENT>if offset == -<NUM_LIT:1>:<EOL><INDENT>offset = fstat.st_size<EOL><DEDENT>if length == -<NUM_LIT:1>:<EOL><INDENT>length = fstat.st_size - offset<EOL><DEDENT>with open(filename, "<STR_LIT:r>") as fp:<EOL><INDENT>fp.seek(offset)<EOL>try:<EOL><INDENT>data = fp.read(length)<EOL><DEDENT>except IOError:<EOL><INDENT>return {}<EOL><DEDENT><DEDENT>if data:<EOL><INDENT>data = _escape_data(data) if escape_data else data<EOL>return dict(offset=offset, length=len(data), data=data)<EOL><DEDENT>return dict(offset=offset, length=<NUM_LIT:0>)<EOL>

Read a chunk of a file from an offset upto the length.


stdin = None if prev_proc is None else prev_proc.stdout<EOL>process = subprocess.Popen(to_cmd,<EOL>stdout=subprocess.PIPE,<EOL>stdin=stdin)<EOL>if prev_proc is not None:<EOL><INDENT>prev_proc.stdout.close() <EOL><DEDENT>return process<EOL>

Pipes output of prev_proc into to_cmd.
Returns piped process


process = subprocess.Popen(cmd, stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE, cwd=cwd, env=env)<EOL>stdout_builder, stderr_builder = proc.async_stdout_stderr_builder(process)<EOL>process.wait()<EOL>stdout, stderr = stdout_builder.result(), stderr_builder.result()<EOL>return {'<STR_LIT>': '<STR_LIT:U+0020>'.join(cmd), '<STR_LIT>': stderr, '<STR_LIT>': stdout}<EOL>

Runs the command and returns its stdout and stderr.


command = '<STR_LIT>'.join(map(lambda x: '<STR_LIT:U+0020>'.join(x), cmd_list))<EOL>chained_proc = functools.reduce(pipe, [None] + cmd_list)<EOL>stdout_builder = proc.async_stdout_builder(chained_proc)<EOL>chained_proc.wait()<EOL>return {<EOL>'<STR_LIT>': command,<EOL>'<STR_LIT>': stdout_builder.result()<EOL>}<EOL>

Feed output of one command to the next and return final output
Returns string output of chained application of commands.


return instance_id.split('<STR_LIT:_>')[<NUM_LIT:1>]<EOL>


return pkgutil.get_data("<STR_LIT>", os.path.join("<STR_LIT>", asset_name))<EOL>


return not path.startswith("<STR_LIT:/>") and "<STR_LIT:..>" not in path<EOL>

file path should be a relative path without ".." in it
:param path: file path
:return: true if the path is relative and doesn't contain ".."


raw = OutgoingPacket.create_packet(reqid, message).raw<EOL>dispatcher = MockDispatcher()<EOL>dispatcher.prepare_with_raw(raw)<EOL>packet = IncomingPacket()<EOL>packet.read(dispatcher)<EOL>packet.data = packet.data<EOL>return packet<EOL>

Convert (reqid, message) pair to IncomingPacket object


pkt_list = []<EOL>raw_list = []<EOL>message_list = [mock_protobuf.get_mock_register_response(),<EOL>mock_protobuf.get_mock_config(),<EOL>mock_protobuf.get_mock_bolt(),<EOL>mock_protobuf.get_mock_topology()]<EOL>for message in message_list:<EOL><INDENT>if is_message:<EOL><INDENT>reqid = REQID.generate_zero()<EOL><DEDENT>else:<EOL><INDENT>reqid = REQID.generate()<EOL><DEDENT>normal_pkt = convert_to_incoming_packet(reqid, message)<EOL>pkt_list.append(normal_pkt)<EOL>raw_list.append((reqid, message))<EOL><DEDENT>return pkt_list, raw_list<EOL>

Returns a list of valid IncomingPackets with non-zero REQID and the corresponding raw data


reqid = REQID.generate()<EOL>message = mock_protobuf.get_mock_register_response()<EOL>pkt = convert_to_incoming_packet(reqid, message)<EOL>return pkt, reqid, message<EOL>

Returns a tuple of mock (IncomingPacket, REQID, RegisterResponse message)


pkt_list = []<EOL>raw_list = mock_protobuf.get_many_mock_pplans()<EOL>for msg in raw_list:<EOL><INDENT>reqid = REQID.generate_zero()<EOL>pkt = convert_to_incoming_packet(reqid, msg)<EOL>pkt_list.append(pkt)<EOL><DEDENT>builder, typename = mock_protobuf.get_pplan_builder_and_typename()<EOL>return pkt_list, raw_list, builder, typename<EOL>

Get a list of IncomingPackets, the corresponding mock messages, builder and typename


raw = OutgoingPacket.create_packet(REQID.generate(), mock_protobuf.get_mock_pplan()).raw<EOL>packet = IncomingPacket.create_packet(raw[:<NUM_LIT:4>], raw[<NUM_LIT:4>:])<EOL>packet.is_complete = False<EOL>return packet<EOL>

Returns an incomplete IncomingPacket object


self.to_be_received = raw<EOL>

the content of ``raw`` will be prepared in the recv buffer


pkt_list, _ = get_mock_requst_packets(is_message=False)<EOL>for pkt in pkt_list:<EOL><INDENT>self.to_be_received += pkt.convert_to_raw()<EOL><DEDENT>

a number of valid packets will be prepared in the recv buffer


pkt = get_mock_requst_packets(is_message=False)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>self.to_be_received = pkt.header<EOL>

a packet with just a header (incomplete packet) will be prepared in the recv buffer


pkt = get_mock_requst_packets(is_message=False)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>self.to_be_received = pkt.header + pkt.data[:self.PARTIAL_DATA_SIZE]<EOL>

a partial data packet will be prepared in the recv buffer


self.eagain_test = True<EOL>

prepare so that EAGAIN error is raised when recv() is called


self.fatal_error_test = True<EOL>

prepare so that RuntimeError is raised when recv() is called


if self.fatal_error_test:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>elif self.eagain_test:<EOL><INDENT>raise socket.error(socket.errno.EAGAIN, "<STR_LIT>")<EOL><DEDENT>ret = self.to_be_received[:numbytes]<EOL>self.to_be_received = self.to_be_received[numbytes:]<EOL>return ret<EOL>


return len(buf)<EOL>

mock sends the content of a given buffer


spout_1 = mock_protobuf.get_mock_spout(component=mock_protobuf.get_mock_component(name="<STR_LIT>"))<EOL>bolt_1 = mock_protobuf.get_mock_bolt(component=mock_protobuf.get_mock_component(name="<STR_LIT>"))<EOL>bolt_2 = mock_protobuf.get_mock_bolt(component=mock_protobuf.get_mock_component(name="<STR_LIT>"))<EOL>topology = mock_protobuf.get_mock_topology(spouts=[spout_1], bolts=[bolt_1, bolt_2])<EOL>instance_ids = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>task_ids = [<NUM_LIT:100>, <NUM_LIT:200>, <NUM_LIT>]<EOL>comp_indexes = [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]<EOL>comp_names = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>instances = []<EOL>for i_id, t_id, c_i, c_name in zip(instance_ids, task_ids, comp_indexes, comp_names):<EOL><INDENT>info = mock_protobuf.get_mock_instance_info(task_id=t_id,<EOL>component_index=c_i,<EOL>component_name=c_name)<EOL>instance = mock_protobuf.get_mock_instance(instance_id=i_id, info=info)<EOL>instances.append(instance)<EOL><DEDENT>pplan = mock_protobuf.get_mock_pplan(topology=topology, instances=instances)<EOL>keys = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>zipped = zip(instance_ids, task_ids, comp_indexes, comp_names)<EOL>return pplan, [dict(zip(keys, z)) for z in zipped]<EOL>

Returns a legitimate looking physical plan

    This topology has 1 spout and 2 bolts. Currently no input/output streams.
    There is only one stream manager.

    [Instance 1: spout1]
      - instance_id = "instance1"
      - task_id = 100
      - component_index = 0
      - component_name = "spout1"

    [Instance 2: bolt1]
      - instance_id = "instance2"
      - task_id = 200
      - component_index = 0
      - component_name = "bolt1"

    [instance 3: bolt2]
      - instance_id = "instance3"
      - task_id = 300
      - component_index = 0
      - component_name = "bolt2"

    :returns: PhysicalPlan message and a list of dictionaries for each instance containing
              (instance_id, task_id, comp_index, comp_name)


data_tuple = tuple_pb2.HeronDataTuple()<EOL>data_tuple.key = <NUM_LIT:0><EOL>tuple_size_in_bytes = <NUM_LIT:0><EOL>for obj in lst:<EOL><INDENT>serialized = serializer.serialize(obj)<EOL>data_tuple.values.append(serialized)<EOL>tuple_size_in_bytes += len(serialized)<EOL><DEDENT>return data_tuple, tuple_size_in_bytes<EOL>

Make HeronDataTuple from a list of objects


if config_dict is None:<EOL><INDENT>return topology_pb2.Config()<EOL><DEDENT>proto_config = topology_pb2.Config()<EOL>config_serializer = PythonSerializer()<EOL>assert isinstance(config_dict, dict)<EOL>for key, value in config_dict.items():<EOL><INDENT>if isinstance(value, bool):<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.value = "<STR_LIT:true>" if value else "<STR_LIT:false>"<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT>elif isinstance(value, (str, int, float)):<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.value = str(value)<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.serialized_value = config_serializer.serialize(value)<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT><DEDENT>return proto_config<EOL>

Returns a protobuf Config object from topology_pb2


component = topology_pb2.Component()<EOL>component.name = name<EOL>component.spec = topology_pb2.ComponentObjectSpec.Value("<STR_LIT>")<EOL>component.class_name = python_cls<EOL>component.config.CopyFrom(config)<EOL>return component<EOL>

Returns a mock protobuf Component object from topology_pb2


stream_id = topology_pb2.StreamId()<EOL>stream_id.id = id<EOL>stream_id.component_name = component_name<EOL>return stream_id<EOL>

Returns a mock protobuf StreamId from topology_pb2


def get_mock_bolt(component=get_mock_component(), inputs=[], outputs=[]):

bolt = topology_pb2.Bolt()<EOL>bolt.comp.CopyFrom(component)<EOL>for i in inputs:<EOL><INDENT>added = bolt.inputs.add()<EOL>added.CopyFrom(i)<EOL><DEDENT>for o in outputs:<EOL><INDENT>added = bolt.outputs.add()<EOL>added.CopyFrom(o)<EOL><DEDENT>return bolt<EOL>

Returns a mock protobuf Bolt object from topology_pb2


spout = topology_pb2.Spout()<EOL>spout.comp.CopyFrom(component)<EOL>for out in outputs:<EOL><INDENT>added = spout.outputs.add()<EOL>added.CopyFrom(out)<EOL><DEDENT>return spout<EOL>

Returns a mock protobuf Spout object from topology_pb2


def get_mock_topology(id="<STR_LIT>", name="<STR_LIT>", state=<NUM_LIT:1>, spouts=[], bolts=[]):

<EOL>topology = topology_pb2.Topology()<EOL>topology.id = id<EOL>topology.name = name<EOL>topology.state = state<EOL>for sp in spouts:<EOL><INDENT>added = topology.spouts.add()<EOL>added.CopyFrom(sp)<EOL><DEDENT>for bl in bolts:<EOL><INDENT>added = topology.bolts.add()<EOL>added.CopyFrom(bl)<EOL><DEDENT>return topology<EOL>

Returns a mock protobuf Topology object from topology_pb2


def get_mock_stmgr(id="<STR_LIT>", host="<STR_LIT:127.0.0.1>", port=<NUM_LIT>, endpoint="<STR_LIT:hello>"):

stmgr = physical_plan_pb2.StMgr()<EOL>stmgr.id = id<EOL>stmgr.host_name = host<EOL>stmgr.data_port = port<EOL>stmgr.local_endpoint = endpoint<EOL>return stmgr<EOL>

Returns a mock protobuf StMgr object from physical_plan_pb2


def get_mock_instance_info(task_id=<NUM_LIT>, component_index=<NUM_LIT>, component_name="<STR_LIT:hello>"):

<EOL>instance_info = physical_plan_pb2.InstanceInfo()<EOL>instance_info.task_id = task_id<EOL>instance_info.component_index = component_index<EOL>instance_info.component_name = component_name<EOL>return instance_info<EOL>

Returns a mock protobuf InstanceInfo object from physical_plan_pb2


instance = physical_plan_pb2.Instance()<EOL>instance.instance_id = instance_id<EOL>instance.stmgr_id = stmgr_id<EOL>instance.info.CopyFrom(info)<EOL>return instance<EOL>

Returns a mock protobuf Instance object from physical_plan_pb2


pplan = physical_plan_pb2.PhysicalPlan()<EOL>pplan.topology.MergeFrom(topology)<EOL>if len(stmgrs) == <NUM_LIT:0>:<EOL><INDENT>stmgrs.append(get_mock_stmgr())<EOL><DEDENT>if len(instances) == <NUM_LIT:0>:<EOL><INDENT>instances.append(get_mock_instance())<EOL><DEDENT>for stmgr in stmgrs:<EOL><INDENT>added = pplan.stmgrs.add()<EOL>added.CopyFrom(stmgr)<EOL><DEDENT>for instance in instances:<EOL><INDENT>added = pplan.instances.add()<EOL>added.CopyFrom(instance)<EOL><DEDENT>return pplan<EOL>

Returns a mock protobuf PhysicalPlan object from physical_plan_pb2


mock_status = common_pb2.Status()<EOL>mock_status.status = common_pb2.StatusCode.Value(status)<EOL>mock_status.message = message<EOL>return mock_status<EOL>

Returns a mock protobuf Status object from common_pb2


<EOL>mock_message = stmgr_pb2.NewInstanceAssignmentMessage()<EOL>mock_message.pplan.MergeFrom(pplan)<EOL>return mock_message<EOL>

Returns a mock protobuf NewInstanceAssignmentMessage object from stmgr_pb2


mock_response = stmgr_pb2.RegisterInstanceResponse()<EOL>mock_response.status.MergeFrom(status)<EOL>mock_response.pplan.MergeFrom(pplan)<EOL>return mock_response<EOL>

Returns a mock protobuf RegisterInstanceResponse object from stmgr_pb2


<EOL>builder = lambda: physical_plan_pb2.PhysicalPlan()<EOL>typename = builder().DESCRIPTOR.full_name<EOL>return builder, typename<EOL>

Returns a PhysicalPlan builder callable and typename 'PhysicalPlan


pplans_lst = []<EOL>for i in range(<NUM_LIT:10>):<EOL><INDENT>_id = "<STR_LIT>" + str(i)<EOL>pplan = get_mock_pplan(stmgrs=[get_mock_stmgr(id=_id)])<EOL>pplans_lst.append(pplan)<EOL><DEDENT>return pplans_lst<EOL>

Returns a list of 10 PhysicalPlan objects, differing just by stream manager id


def emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID,<EOL>direct_task=None, need_task_ids=False):

<EOL>self.pplan_helper.check_output_schema(stream, tup)<EOL>custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)<EOL>self.pplan_helper.context.invoke_hook_emit(tup, stream, None)<EOL>data_tuple = tuple_pb2.HeronDataTuple()<EOL>data_tuple.key = <NUM_LIT:0><EOL>if direct_task is not None:<EOL><INDENT>if not isinstance(direct_task, int):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(direct_task)))<EOL><DEDENT>data_tuple.dest_task_ids.append(direct_task)<EOL><DEDENT>elif custom_target_task_ids is not None:<EOL><INDENT>for task_id in custom_target_task_ids:<EOL><INDENT>data_tuple.dest_task_ids.append(task_id)<EOL><DEDENT><DEDENT>if tup_id is not None:<EOL><INDENT>tuple_info = TupleHelper.make_root_tuple_info(stream, tup_id)<EOL>if self.acking_enabled:<EOL><INDENT>root = data_tuple.roots.add()<EOL>root.taskid = self.pplan_helper.my_task_id<EOL>root.key = tuple_info.key<EOL>self.in_flight_tuples[tuple_info.key] = tuple_info<EOL><DEDENT>else:<EOL><INDENT>self.immediate_acks.append(tuple_info)<EOL><DEDENT><DEDENT>tuple_size_in_bytes = <NUM_LIT:0><EOL>start_time = time.time()<EOL>for obj in tup:<EOL><INDENT>serialized = self.serializer.serialize(obj)<EOL>data_tuple.values.append(serialized)<EOL>tuple_size_in_bytes += len(serialized)<EOL><DEDENT>serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS<EOL>self.spout_metrics.serialize_data_tuple(stream, serialize_latency_ns)<EOL>super(SpoutInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,<EOL>tuple_size_in_bytes=tuple_size_in_bytes)<EOL>self.total_tuples_emitted += <NUM_LIT:1><EOL>self.spout_metrics.update_emit_count(stream)<EOL>if need_task_ids:<EOL><INDENT>sent_task_ids = custom_target_task_ids or []<EOL>if direct_task is not None:<EOL><INDENT>sent_task_ids.append(direct_task)<EOL><DEDENT>return sent_task_ids<EOL><DEDENT>

Emits a new tuple from this Spout

        It is compatible with StreamParse API.

        :type tup: list or tuple
        :param tup: the new output Tuple to send from this spout,
                    should contain only serializable data.
        :type tup_id: str or object
        :param tup_id: the ID for the Tuple. Leave this blank for an unreliable emit.
                       (Same as messageId in Java)
        :type stream: str
        :param stream: the ID of the stream this Tuple should be emitted to.
                       Leave empty to emit to the default stream.
        :type direct_task: int
        :param direct_task: the task to send the Tuple to if performing a direct emit.
        :type need_task_ids: bool
        :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.


return self._is_topology_running() and self.output_helper.is_out_queue_available()<EOL>

Checks whether we could produce tuples, i.e. invoke spout.next_tuple()


if not self._is_topology_running():<EOL><INDENT>return False<EOL><DEDENT>max_spout_pending =self.pplan_helper.context.get_cluster_config().get(api_constants.TOPOLOGY_MAX_SPOUT_PENDING)<EOL>if not self.acking_enabled and self.output_helper.is_out_queue_available():<EOL><INDENT>return True<EOL><DEDENT>elif self.acking_enabled and self.output_helper.is_out_queue_available() andlen(self.in_flight_tuples) < max_spout_pending:<EOL><INDENT>return True<EOL><DEDENT>elif self.acking_enabled and not self.in_stream.is_empty():<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Checks whether we still need to do more work

        When the topology state is RUNNING:
        1. if the out_queue is not full and ack is not enabled, we could wake up next time to
           produce more tuples and push to the out_queue
        2. if the out_queue is not full but the acking is enabled, we need to make sure that
          the number of pending tuples is smaller than max_spout_pending
        3. if there are more to read, we will wake up itself next time.


if level is None:<EOL><INDENT>_log_level = logging.INFO<EOL><DEDENT>else:<EOL><INDENT>if level == "<STR_LIT>" or level == "<STR_LIT>":<EOL><INDENT>_log_level = logging.DEBUG<EOL><DEDENT>elif level == "<STR_LIT:info>":<EOL><INDENT>_log_level = logging.INFO<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>_log_level = logging.WARNING<EOL><DEDENT>elif level == "<STR_LIT:error>":<EOL><INDENT>_log_level = logging.ERROR<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(level))<EOL><DEDENT><DEDENT>self.logger.log(_log_level, message)<EOL>

Log message, optionally providing a logging level

        It is compatible with StreamParse API.

        :type message: str
        :param message: the log message to send
        :type level: str
        :param level: the logging level,
                      one of: trace (=debug), debug, info, warn or error (default: info)


try:<EOL><INDENT>if is_spout:<EOL><INDENT>spout_proto = self.pplan_helper.get_my_spout()<EOL>py_classpath = spout_proto.comp.class_name<EOL>self.logger.info("<STR_LIT>", py_classpath)<EOL><DEDENT>else:<EOL><INDENT>bolt_proto = self.pplan_helper.get_my_bolt()<EOL>py_classpath = bolt_proto.comp.class_name<EOL>self.logger.info("<STR_LIT>", py_classpath)<EOL><DEDENT>pex_loader.load_pex(self.pplan_helper.topology_pex_abs_path)<EOL>spbl_class = pex_loader.import_and_get_class(self.pplan_helper.topology_pex_abs_path,<EOL>py_classpath)<EOL><DEDENT>except Exception as e:<EOL><INDENT>spbl = "<STR_LIT>" if is_spout else "<STR_LIT>"<EOL>self.logger.error(traceback.format_exc())<EOL>raise RuntimeError("<STR_LIT>" % (spbl, str(e)))<EOL><DEDENT>return spbl_class<EOL>

Loads user defined component (spout/bolt)


raise NotImplementedError()<EOL>


raise NotImplementedError()<EOL>

Do the basic clean for Heron Instance

        Note that this method is not guaranteed to be invoked


raise NotImplementedError()<EOL>

Should be called when a tuple was buffered into in_stream