def emit(self, tup, stream=Stream.DEFAULT_STREAM_ID,<EOL>anchors=None, direct_task=None, need_task_ids=False):

<EOL>self.pplan_helper.check_output_schema(stream, tup)<EOL>custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)<EOL>self.pplan_helper.context.invoke_hook_emit(tup, stream, None)<EOL>data_tuple = tuple_pb2.HeronDataTuple()<EOL>data_tuple.key = <NUM_LIT:0><EOL>if direct_task is not None:<EOL><INDENT>if not isinstance(direct_task, int):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(direct_task)))<EOL><DEDENT>data_tuple.dest_task_ids.append(direct_task)<EOL><DEDENT>elif custom_target_task_ids is not None:<EOL><INDENT>for task_id in custom_target_task_ids:<EOL><INDENT>data_tuple.dest_task_ids.append(task_id)<EOL><DEDENT><DEDENT>if anchors is not None:<EOL><INDENT>merged_roots = set()<EOL>for tup in [t for t in anchors if isinstance(t, HeronTuple) and t.roots is not None]:<EOL><INDENT>merged_roots.update(tup.roots)<EOL><DEDENT>for rt in merged_roots:<EOL><INDENT>to_add = data_tuple.roots.add()<EOL>to_add.CopyFrom(rt)<EOL><DEDENT><DEDENT>tuple_size_in_bytes = <NUM_LIT:0><EOL>start_time = time.time()<EOL>for obj in tup:<EOL><INDENT>serialized = self.serializer.serialize(obj)<EOL>data_tuple.values.append(serialized)<EOL>tuple_size_in_bytes += len(serialized)<EOL><DEDENT>serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS<EOL>self.bolt_metrics.serialize_data_tuple(stream, serialize_latency_ns)<EOL>super(BoltInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,<EOL>tuple_size_in_bytes=tuple_size_in_bytes)<EOL>self.bolt_metrics.update_emit_count(stream)<EOL>if need_task_ids:<EOL><INDENT>sent_task_ids = custom_target_task_ids or []<EOL>if direct_task is not None:<EOL><INDENT>sent_task_ids.append(direct_task)<EOL><DEDENT>return sent_task_ids<EOL><DEDENT>

Emits a new tuple from this Bolt

        It is compatible with StreamParse API.

        :type tup: list or tuple
        :param tup: the new output Tuple to send from this bolt,
                    should only contain only serializable data.
        :type stream: str
        :param stream: the ID of the stream to emit this Tuple to.
                       Leave empty to emit to the default stream.
        :type anchors: list
        :param anchors: a list of HeronTuples to which the emitted Tuples should be anchored.
        :type direct_task: int
        :param direct_task: the task to send the Tupel to if performing a direct emit.
        :type need_task_ids: bool
        :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.


Should be called when tuple was buffered into in_stream

        This method is equivalent to ``addBoltTasks()`` but
        is designed for event-driven single-thread bolt.


if not isinstance(tup, HeronTuple):<EOL><INDENT>Log.error("<STR_LIT>")<EOL>return<EOL><DEDENT>if self.acking_enabled:<EOL><INDENT>ack_tuple = tuple_pb2.AckTuple()<EOL>ack_tuple.ackedtuple = int(tup.id)<EOL>tuple_size_in_bytes = <NUM_LIT:0><EOL>for rt in tup.roots:<EOL><INDENT>to_add = ack_tuple.roots.add()<EOL>to_add.CopyFrom(rt)<EOL>tuple_size_in_bytes += rt.ByteSize()<EOL><DEDENT>super(BoltInstance, self).admit_control_tuple(ack_tuple, tuple_size_in_bytes, True)<EOL><DEDENT>process_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS<EOL>self.pplan_helper.context.invoke_hook_bolt_ack(tup, process_latency_ns)<EOL>self.bolt_metrics.acked_tuple(tup.stream, tup.component, process_latency_ns)<EOL>

Indicate that processing of a Tuple has succeeded

        It is compatible with StreamParse API.


if not isinstance(tup, HeronTuple):<EOL><INDENT>Log.error("<STR_LIT>")<EOL>return<EOL><DEDENT>if self.acking_enabled:<EOL><INDENT>fail_tuple = tuple_pb2.AckTuple()<EOL>fail_tuple.ackedtuple = int(tup.id)<EOL>tuple_size_in_bytes = <NUM_LIT:0><EOL>for rt in tup.roots:<EOL><INDENT>to_add = fail_tuple.roots.add()<EOL>to_add.CopyFrom(rt)<EOL>tuple_size_in_bytes += rt.ByteSize()<EOL><DEDENT>super(BoltInstance, self).admit_control_tuple(fail_tuple, tuple_size_in_bytes, False)<EOL><DEDENT>fail_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS<EOL>self.pplan_helper.context.invoke_hook_bolt_fail(tup, fail_latency_ns)<EOL>self.bolt_metrics.failed_tuple(tup.stream, tup.component, fail_latency_ns)<EOL>

Indicate that processing of a Tuple has failed

        It is compatible with StreamParse API.


sys_config = system_config.get_sys_config()<EOL>opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,<EOL>const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,<EOL>const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,<EOL>const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,<EOL>const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,<EOL>const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]<EOL>Log.debug("<STR_LIT>")<EOL>try:<EOL><INDENT>value_lst = [int(sys_config[opt]) for opt in opt_list]<EOL>sock_opt = SocketOptions(*value_lst)<EOL>return sock_opt<EOL><DEDENT>except ValueError as e:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(e))<EOL><DEDENT>except KeyError as e:<EOL><INDENT>raise KeyError("<STR_LIT>" % str(e))<EOL><DEDENT>

Creates SocketOptions object from a given sys_config dict


def __init__(self, looper, hostname, port, socket_map, socket_options):

asyncore.dispatcher.__init__(self, map=socket_map)<EOL>self.looper = looper<EOL>self.hostname = hostname<EOL>self.port = int(port)<EOL>self.endpoint = (self.hostname, self.port)<EOL>self.out_buffer = []<EOL>self.socket_options = socket_options<EOL>self.registered_message_map = dict()<EOL>self.response_message_map = dict()<EOL>self.context_map = dict()<EOL>self.incomplete_pkt = None<EOL>self.total_bytes_written = <NUM_LIT:0><EOL>self.total_pkt_written = <NUM_LIT:0><EOL>self.total_bytes_received = <NUM_LIT:0><EOL>self.total_pkt_received = <NUM_LIT:0><EOL>self._connecting = False<EOL>Log.debug("<STR_LIT>"<EOL>% (self._get_classname(), str(self.endpoint),<EOL>str(socket_map), str(self.socket_options)))<EOL>

Initializes HeronClient

        :type looper: ``GatewayLooper`` (heron.instance.src.python.network)
        :param looper: looper object
        :type hostname: str
        :param hostname: endpoint hostname
        :type port: int
        :param port: endpoint port
        :type socket_map: dict
        :param socket_map: socket map used for asyncore.dispatcher
        :type socket_options: ``SocketOptions`` (heron.common.src.python.network)
        :param socket_options: options for the socket and this client


Log.debug("<STR_LIT>" % self._get_classname())<EOL>self.create_socket(socket.AF_INET, socket.SOCK_STREAM)<EOL>self._connecting = True<EOL>self.connect(self.endpoint)<EOL>

Tries to connect to the Heron Server

        ``loop()`` method needs to be called after this.


message = msg_builder()<EOL>Log.debug("<STR_LIT>" % message.DESCRIPTOR.full_name)<EOL>self.registered_message_map[message.DESCRIPTOR.full_name] = msg_builder<EOL>

Registers protobuf message builders that this client wants to receive

        :param msg_builder: callable to create a protobuf message that this client wants to receive


def send_request(self, request, context, response_type, timeout_sec):

<EOL>reqid = REQID.generate()<EOL>Log.debug("<STR_LIT>" % (self._get_classname(), str(reqid)))<EOL>self.response_message_map[reqid] = response_type<EOL>self.context_map[reqid] = context<EOL>if timeout_sec > <NUM_LIT:0>:<EOL><INDENT>def timeout_task():<EOL><INDENT>self.handle_timeout(reqid)<EOL><DEDENT>self.looper.register_timer_task_in_sec(timeout_task, timeout_sec)<EOL><DEDENT>outgoing_pkt = OutgoingPacket.create_packet(reqid, request)<EOL>self._send_packet(outgoing_pkt)<EOL>

Sends a request message (REQID is non-zero)


Log.debug("<STR_LIT>" % self._get_classname())<EOL>outgoing_pkt = OutgoingPacket.create_packet(REQID.generate_zero(), message)<EOL>self._send_packet(outgoing_pkt)<EOL>


if reqid in self.context_map:<EOL><INDENT>context = self.context_map.pop(reqid)<EOL>self.response_message_map.pop(reqid)<EOL>self.on_response(StatusCode.TIMEOUT_ERROR, context, None)<EOL><DEDENT>


Pushes a packet to a send buffer, the content of which will be send when available


pass<EOL>

Called when the client is connected

        Should be implemented by a subclass.


@abstractmethod<EOL><INDENT>def on_response(self, status, context, response):<DEDENT>

pass<EOL>

Called when the client receives a response

        Should be implemented by a subclass.


pass<EOL>

Called when the client receives a message

        Should be implemented by a subclass.


pass<EOL>

Called when the client meets errors

        Note that this method is not called when a connection is not yet established.
        In such a case, ``on_connect()`` with status == StatusCode.CONNECT_ERROR is called.


while not self.should_exit:<EOL><INDENT>self._run_once()<EOL><DEDENT>self.on_exit()<EOL>

Start loop

        This will start a while loop until ``exit_loop()`` is called.


try:<EOL><INDENT>self.do_wait()<EOL>self._execute_wakeup_tasks()<EOL>self._trigger_timers()<EOL><DEDENT>except Exception as e:<EOL><INDENT>Log.error("<STR_LIT>" + str(e))<EOL>Log.error(traceback.format_exc())<EOL>self.should_exit = True<EOL><DEDENT>

Run once, should be called only from loop()


Log.info("<STR_LIT>")<EOL>for task in self.exit_tasks:<EOL><INDENT>task()<EOL><DEDENT>


pass<EOL>

Blocking operation, should be implemented by a subclass


pass<EOL>

Wakes up do_wait() operation, should be implemented by a subclass

        Note that this method should be implemented in a thread-safe way.


self.wakeup_tasks.append(task)<EOL>self.wake_up()<EOL>

Add a wakeup task

        :param task: function to be run as a wakeup task


Add an exit task

        :param task: function to be run as an exit task


def register_timer_task_in_sec(self, task, second):

<EOL>second_in_float = float(second)<EOL>expiration = time.time() + second_in_float<EOL>heappush(self.timer_tasks, (expiration, task))<EOL>

Registers a new timer task

        :param task: function to be run at a specified second from now
        :param second: how many seconds to wait before the timer is triggered


self.should_exit = True<EOL>self.wake_up()<EOL>


if len(self.timer_tasks) == <NUM_LIT:0>:<EOL><INDENT>return sys.maxsize<EOL><DEDENT>else:<EOL><INDENT>next_timeout_interval = self.timer_tasks[<NUM_LIT:0>][<NUM_LIT:0>] - time.time()<EOL>return next_timeout_interval<EOL><DEDENT>

Get the next timeout from now

        This should be used from do_wait().
        :returns (float) next_timeout, or 10.0 if there are no timer events


<EOL>size = len(self.wakeup_tasks)<EOL>for i in range(size):<EOL><INDENT>self.wakeup_tasks[i]()<EOL><DEDENT>

Executes wakeup tasks, should only be called from loop()


current = time.time()<EOL>while len(self.timer_tasks) > <NUM_LIT:0> and (self.timer_tasks[<NUM_LIT:0>][<NUM_LIT:0>] - current <= <NUM_LIT:0>):<EOL><INDENT>task = heappop(self.timer_tasks)[<NUM_LIT:1>]<EOL>task()<EOL><DEDENT>


if response.status.status != common_pb2.StatusCode.Value("<STR_LIT:OK>"):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>self.is_registered = True<EOL>if response.HasField("<STR_LIT>"):<EOL><INDENT>Log.info("<STR_LIT>")<EOL>self._handle_assignment_message(response.pplan)<EOL><DEDENT>else:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT>

Called when a register response (RegisterInstanceResponse) arrives


self.heron_instance_cls.handle_new_tuple_set_2(hts2)<EOL>


self.heron_instance_cls.handle_initiate_stateful_checkpoint(ckptmsg)<EOL>

Called when new InitiateStatefulCheckpoint arrives


self.heron_instance_cls.handle_start_stateful_processing(startmsg)<EOL>

Called when new StartInstanceStatefulProcessing arrives


self.heron_instance_cls.handle_restore_instance_state(restoremsg)<EOL>

Called when new RestoreInstanceStateRequest arrives


Log.debug("<STR_LIT>", str(pplan))<EOL>self.heron_instance_cls.handle_assignment_msg(pplan)<EOL>

Called when new NewInstanceAssignmentMessage arrives


return struct.pack(HeronProtocol.INT_PACK_FMT, i)<EOL>


return struct.unpack(HeronProtocol.INT_PACK_FMT, i)[<NUM_LIT:0>]<EOL>


return <NUM_LIT:4> + len(string)<EOL>

Get size to pack string, four byte used for specifying length of the string


return <NUM_LIT:4> + message.ByteSize()<EOL>

Get size to pack message, four byte used for specifying length of the message


packet = IncomingPacket()<EOL>packet.read(dispatcher)<EOL>return packet<EOL>

Reads new packet and returns an IncomingPacket object

        :param dispatcher: asyncore's dispatcher class from which packets is read


if not packet.is_complete:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>data = packet.data<EOL>len_typename = HeronProtocol.unpack_int(data[:<NUM_LIT:4>])<EOL>data = data[<NUM_LIT:4>:]<EOL>typename = data[:len_typename]<EOL>data = data[len_typename:]<EOL>reqid = REQID.unpack(data[:REQID.REQID_SIZE])<EOL>data = data[REQID.REQID_SIZE:]<EOL>len_msg = HeronProtocol.unpack_int(data[:<NUM_LIT:4>])<EOL>data = data[<NUM_LIT:4>:]<EOL>serialized_msg = data[:len_msg]<EOL>return typename, reqid, serialized_msg<EOL>

Decodes an IncomingPacket object and returns (typename, reqid, serialized message)


assert message.IsInitialized()<EOL>packet = '<STR_LIT>'<EOL>typename = message.DESCRIPTOR.full_name<EOL>datasize = HeronProtocol.get_size_to_pack_string(typename) +REQID.REQID_SIZE + HeronProtocol.get_size_to_pack_message(message)<EOL>packet += HeronProtocol.pack_int(datasize)<EOL>packet += HeronProtocol.pack_int(len(typename))<EOL>packet += typename<EOL>packet += reqid.pack()<EOL>packet += HeronProtocol.pack_int(message.ByteSize())<EOL>packet += message.SerializeToString()<EOL>return OutgoingPacket(packet)<EOL>

Creates Outgoing Packet from a given reqid and message

        :param reqid: REQID object
        :param message: protocol buffer object


return len(self.to_send) == <NUM_LIT:0><EOL>

Indicates whether this packet is successfully sent


if self.sent_complete:<EOL><INDENT>return<EOL><DEDENT>sent = dispatcher.send(self.to_send)<EOL>self.to_send = self.to_send[sent:]<EOL>

Sends this outgoing packet to dispatcher's socket


self.header = '<STR_LIT>'<EOL>self.data = '<STR_LIT>'<EOL>self.is_header_read = False<EOL>self.is_complete = False<EOL>self.id = random.getrandbits(<NUM_LIT:8>)<EOL>


packet = IncomingPacket()<EOL>packet.header = header<EOL>packet.data = data<EOL>if len(header) == HeronProtocol.HEADER_SIZE:<EOL><INDENT>packet.is_header_read = True<EOL>if len(data) == packet.get_datasize():<EOL><INDENT>packet.is_complete = True<EOL><DEDENT><DEDENT>return packet<EOL>

Creates an IncomingPacket object from header and data

        This method is for testing purposes


return self.header + self.data<EOL>

Converts this IncomingPacket object to raw string

        This method is for testing purposes


if not self.is_header_read:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>return HeronProtocol.unpack_int(self.header)<EOL>

Returns the datasize of the packet

        :returns: (int) datasize of the packet, or -1 if header is incomplete


return len(self.data) + len(self.header)<EOL>

Returns the size of this packet, including header and data


try:<EOL><INDENT>if not self.is_header_read:<EOL><INDENT>to_read = HeronProtocol.HEADER_SIZE - len(self.header)<EOL>self.header += dispatcher.recv(to_read)<EOL>if len(self.header) == HeronProtocol.HEADER_SIZE:<EOL><INDENT>self.is_header_read = True<EOL><DEDENT>else:<EOL><INDENT>Log.debug("<STR_LIT>" % len(self.header))<EOL>return<EOL><DEDENT><DEDENT>if self.is_header_read and not self.is_complete:<EOL><INDENT>to_read = self.get_datasize() - len(self.data)<EOL>self.data += dispatcher.recv(to_read)<EOL>if len(self.data) == self.get_datasize():<EOL><INDENT>self.is_complete = True<EOL><DEDENT><DEDENT><DEDENT>except socket.error as e:<EOL><INDENT>if e.errno == socket.errno.EAGAIN or e.errno == socket.errno.EWOULDBLOCK:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>

Reads incoming data from asyncore.dispatcher


data_bytes = bytearray(random.getrandbits(<NUM_LIT:8>) for i in range(REQID.REQID_SIZE))<EOL>return REQID(data_bytes)<EOL>


data_bytes = bytearray(<NUM_LIT:0> for i in range(REQID.REQID_SIZE))<EOL>return REQID(data_bytes)<EOL>


return self.bytes<EOL>


return self == REQID.generate_zero()<EOL>


return REQID(bytearray(raw_data))<EOL>

Unpacks a given bytestring and returns REQID object


super(GatewayLooper, self).__init__()<EOL>self.sock_map = socket_map<EOL>self.pipe_r, self.pipe_w = os.pipe()<EOL>self.started = time.time()<EOL>Log.debug("<STR_LIT>" + str(time.asctime()))<EOL>

Initializes a GatewayLooper instance

        :param socket_map: socket map used for asyncore.dispatcher


if self.sock_map is None:<EOL><INDENT>Log.warning("<STR_LIT>")<EOL><DEDENT>readable_lst = []<EOL>writable_lst = []<EOL>error_lst = []<EOL>if self.sock_map is not None:<EOL><INDENT>for fd, obj in self.sock_map.items():<EOL><INDENT>is_r = obj.readable()<EOL>is_w = obj.writable()<EOL>if is_r:<EOL><INDENT>readable_lst.append(fd)<EOL><DEDENT>if is_w and not obj.accepting:<EOL><INDENT>writable_lst.append(fd)<EOL><DEDENT>if is_r or is_w:<EOL><INDENT>error_lst.append(fd)<EOL><DEDENT><DEDENT><DEDENT>readable_lst.append(self.pipe_r)<EOL>Log.debug("<STR_LIT>" + str(timeout) + "<STR_LIT>" + str(self.sock_map))<EOL>try:<EOL><INDENT>readable_lst, writable_lst, error_lst =select.select(readable_lst, writable_lst, error_lst, timeout)<EOL><DEDENT>except select.error as err:<EOL><INDENT>Log.debug("<STR_LIT>" + str(err))<EOL>if err.args[<NUM_LIT:0>] != errno.EINTR:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return<EOL><DEDENT><DEDENT>Log.debug("<STR_LIT>" + str(readable_lst) +<EOL>"<STR_LIT>" + str(writable_lst) + "<STR_LIT>" + str(error_lst))<EOL>if self.pipe_r in readable_lst:<EOL><INDENT>Log.debug("<STR_LIT>")<EOL>os.read(self.pipe_r, <NUM_LIT>)<EOL>readable_lst.remove(self.pipe_r)<EOL><DEDENT>if self.sock_map is not None:<EOL><INDENT>for fd in readable_lst:<EOL><INDENT>obj = self.sock_map.get(fd)<EOL>if obj is None:<EOL><INDENT>continue<EOL><DEDENT>asyncore.read(obj)<EOL><DEDENT>for fd in writable_lst:<EOL><INDENT>obj = self.sock_map.get(fd)<EOL>if obj is None:<EOL><INDENT>continue<EOL><DEDENT>asyncore.write(obj)<EOL><DEDENT>for fd in error_lst:<EOL><INDENT>obj = self.sock_map.get(fd)<EOL>if obj is None:<EOL><INDENT>continue<EOL><DEDENT>asyncore._exception(obj)<EOL><DEDENT><DEDENT>

Modified version of poll() from asyncore module


@staticmethod<EOL><INDENT>def make_tuple(stream, tuple_key, values, roots=None):<DEDENT>

component_name = stream.component_name<EOL>stream_id = stream.id<EOL>gen_task = roots[<NUM_LIT:0>].taskid if roots is not None and len(roots) > <NUM_LIT:0> else None<EOL>return HeronTuple(id=str(tuple_key), component=component_name, stream=stream_id,<EOL>task=gen_task, values=values, creation_time=time.time(), roots=roots)<EOL>

Creates a HeronTuple

        :param stream: protobuf message ``StreamId``
        :param tuple_key: tuple id
        :param values: a list of values
        :param roots: a list of protobuf message ``RootId``


return HeronTuple(id=TupleHelper.TICK_TUPLE_ID, component=TupleHelper.TICK_SOURCE_COMPONENT,<EOL>stream=TupleHelper.TICK_TUPLE_ID, task=None, values=None,<EOL>creation_time=time.time(), roots=None)<EOL>


key = random.getrandbits(TupleHelper.MAX_SFIXED64_RAND_BITS)<EOL>return RootTupleInfo(stream_id=stream_id, tuple_id=tuple_id,<EOL>insertion_time=time.time(), key=key)<EOL>


def add(self, stream_id, task_ids, grouping, source_comp_name):

if stream_id not in self.targets:<EOL><INDENT>self.targets[stream_id] = []<EOL><DEDENT>self.targets[stream_id].append(Target(task_ids, grouping, source_comp_name))<EOL>

Adds the target component

        :type stream_id: str
        :param stream_id: stream id into which tuples are emitted
        :type task_ids: list of str
        :param task_ids: list of task ids to which tuples are emitted
        :type grouping: ICustomStreamGrouping object
        :param grouping: custom grouping to use
        :type source_comp_name: str
        :param source_comp_name: source component name


for stream_id, targets in self.targets.items():<EOL><INDENT>for target in targets:<EOL><INDENT>target.prepare(context, stream_id)<EOL><DEDENT><DEDENT>

Prepares the custom grouping for this component


def choose_tasks(self, stream_id, values):

if stream_id not in self.targets:<EOL><INDENT>return []<EOL><DEDENT>ret = []<EOL>for target in self.targets[stream_id]:<EOL><INDENT>ret.extend(target.choose_tasks(values))<EOL><DEDENT>return ret<EOL>

Choose tasks for a given stream_id and values and Returns a list of target tasks


self.grouping.prepare(context, self.source_comp_name, stream_id, self.task_ids)<EOL>

Invoke prepare() of this custom grouping


ret = self.grouping.choose_tasks(values)<EOL>if not isinstance(ret, list):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>" % str(type(ret)))<EOL><DEDENT>else:<EOL><INDENT>for i in ret:<EOL><INDENT>if not isinstance(i, int):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>" % str(i))<EOL><DEDENT>if i not in self.task_ids:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>" % i)<EOL><DEDENT><DEDENT>return ret<EOL><DEDENT>

Invoke choose_tasks() of this custom grouping


Sends out currently buffered tuples into the Out-Stream


def add_data_tuple(self, stream_id, new_data_tuple, tuple_size_in_bytes):

if (self.current_data_tuple_set is None) or(self.current_data_tuple_set.stream.id != stream_id) or(len(self.current_data_tuple_set.tuples) >= self.data_tuple_set_capacity) or(self.current_data_tuple_size_in_bytes >= self.max_data_tuple_size_in_bytes):<EOL><INDENT>self._init_new_data_tuple(stream_id)<EOL><DEDENT>added_tuple = self.current_data_tuple_set.tuples.add()<EOL>added_tuple.CopyFrom(new_data_tuple)<EOL>self.current_data_tuple_size_in_bytes += tuple_size_in_bytes<EOL>self.total_data_emitted_in_bytes += tuple_size_in_bytes<EOL>

Add a new data tuple to the currently buffered set of tuples


def add_control_tuple(self, new_control_tuple, tuple_size_in_bytes, is_ack):

if self.current_control_tuple_set is None:<EOL><INDENT>self._init_new_control_tuple()<EOL><DEDENT>elif is_ack and (len(self.current_control_tuple_set.fails) > <NUM_LIT:0> or<EOL>len(self.current_control_tuple_set.acks) >= self.control_tuple_set_capacity):<EOL><INDENT>self._init_new_control_tuple()<EOL><DEDENT>elif not is_ack and(len(self.current_control_tuple_set.acks) > <NUM_LIT:0> or<EOL>len(self.current_control_tuple_set.fails) >= self.control_tuple_set_capacity):<EOL><INDENT>self._init_new_control_tuple()<EOL><DEDENT>if is_ack:<EOL><INDENT>added_tuple = self.current_control_tuple_set.acks.add()<EOL><DEDENT>else:<EOL><INDENT>added_tuple = self.current_control_tuple_set.fails.add()<EOL><DEDENT>added_tuple.CopyFrom(new_control_tuple)<EOL>self.total_data_emitted_in_bytes += tuple_size_in_bytes<EOL>

Add a new control (Ack/Fail) tuple to the currently buffered set of tuples

        :param is_ack: ``True`` if Ack, ``False`` if Fail


def add_ckpt_state(self, ckpt_id, ckpt_state):

<EOL>self._flush_remaining()<EOL>msg = ckptmgr_pb2.StoreInstanceStateCheckpoint()<EOL>istate = ckptmgr_pb2.InstanceStateCheckpoint()<EOL>istate.checkpoint_id = ckpt_id<EOL>istate.state = ckpt_state<EOL>msg.state.CopyFrom(istate)<EOL>self._push_tuple_to_stream(msg)<EOL>

Add the checkpoint state message to be sent back the stmgr

        :param ckpt_id: The id of the checkpoint
        :ckpt_state: The checkpoint state


self._flush_remaining()<EOL>self.out_stream.clear()<EOL>


cluster_config = context.get_cluster_config()<EOL>serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None)<EOL>if serializer_clsname is None:<EOL><INDENT>return PythonSerializer()<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>topo_pex_path = context.get_topology_pex_path()<EOL>pex_loader.load_pex(topo_pex_path)<EOL>serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname)<EOL>serializer = serializer_cls()<EOL>return serializer<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>% (serializer_clsname, str(e)))<EOL><DEDENT><DEDENT>

Returns a serializer for a given context


def __init__(self, producer_cb=None, consumer_cb=None):

self._producer_callback = producer_cb<EOL>self._consumer_callback = consumer_cb<EOL>self._buffer = Queue.Queue()<EOL>self.capacity = sys.maxsize<EOL>

Initialize HeronCommunicator

        :param producer_cb: Callback function to be called (usually on producer thread)
               when ``poll()`` is called by the consumer. Default ``None``
        :param consumer_cb: Callback function to be called (usually on consumer thread)
               when ``offer()`` is called by the producer. Default ``None``


self.capacity = capacity<EOL>

Registers the capacity of this communicator

        By default, the capacity of HeronCommunicator is set to be ``sys.maxsize``


return self._buffer.qsize()<EOL>


return self._buffer.empty()<EOL>


try:<EOL><INDENT>ret = self._buffer.get(block=False)<EOL>if self._producer_callback is not None:<EOL><INDENT>self._producer_callback()<EOL><DEDENT>return ret<EOL><DEDENT>except Queue.Empty:<EOL><INDENT>Log.debug("<STR_LIT>" % str(self))<EOL>raise Queue.Empty<EOL><DEDENT>

Poll from the buffer

        It is a non-blocking operation, and when the buffer is empty, it raises Queue.Empty exception


try:<EOL><INDENT>self._buffer.put(item, block=False)<EOL>if self._consumer_callback is not None:<EOL><INDENT>self._consumer_callback()<EOL><DEDENT>return True<EOL><DEDENT>except Queue.Full:<EOL><INDENT>Log.debug("<STR_LIT>" % str(self))<EOL>raise Queue.Full<EOL><DEDENT>

Offer to the buffer

        It is a non-blocking operation, and when the buffer is full, it raises Queue.Full exception


while not self.is_empty():<EOL><INDENT>self.poll()<EOL><DEDENT>


def check_output_schema(self, stream_id, tup):

<EOL>size = self._output_schema.get(stream_id, None)<EOL>if size is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>% (self.my_component_name, stream_id))<EOL><DEDENT>elif size != len(tup):<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (stream_id, size, len(tup)))<EOL><DEDENT>

Checks if a given stream_id and tuple matches with the output schema

        :type stream_id: str
        :param stream_id: stream id into which tuple is sent
        :type tup: list
        :param tup: tuple that is going to be sent


if self.is_spout:<EOL><INDENT>return self._my_spbl<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Returns spout instance, or ``None`` if bolt is assigned


if self.is_spout:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return self._my_spbl<EOL><DEDENT>

Returns bolt instance, or ``None`` if spout is assigned


return self.pplan.topology.state<EOL>


return self.pplan.topology.state == topology_pb2.TopologyState.Value("<STR_LIT>")<EOL>

Checks whether topology is currently running


Checks whether topology is currently paused


Checks whether topology is already killed


if self.pplan.topology.HasField("<STR_LIT>"):<EOL><INDENT>return self._get_dict_from_config(self.pplan.topology.topology_config)<EOL><DEDENT>else:<EOL><INDENT>return {}<EOL><DEDENT>


Log.debug("<STR_LIT>")<EOL>cluster_config = self.get_topology_config()<EOL>cluster_config.update(self._get_dict_from_config(self.my_component.config))<EOL>task_to_component_map = self._get_task_to_comp_map()<EOL>self.context = TopologyContextImpl(cluster_config, self.pplan.topology, task_to_component_map,<EOL>self.my_task_id, metrics_collector,<EOL>self.topology_pex_abs_path)<EOL>


config = {}<EOL>for kv in topology_config.kvs:<EOL><INDENT>if kv.HasField("<STR_LIT:value>"):<EOL><INDENT>assert kv.type == topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL>if PhysicalPlanHelper._is_number(kv.value):<EOL><INDENT>config[kv.key] = PhysicalPlanHelper._get_number(kv.value)<EOL><DEDENT>elif kv.value.lower() in ("<STR_LIT:true>", "<STR_LIT:false>"):<EOL><INDENT>config[kv.key] = True if kv.value.lower() == "<STR_LIT:true>" else False<EOL><DEDENT>else:<EOL><INDENT>config[kv.key] = kv.value<EOL><DEDENT><DEDENT>elif kv.HasField("<STR_LIT>") andkv.type == topology_pb2.ConfigValueType.Value("<STR_LIT>"):<EOL><INDENT>config[kv.key] = default_serializer.deserialize(kv.serialized_value)<EOL><DEDENT>else:<EOL><INDENT>assert kv.HasField("<STR_LIT:type>")<EOL>Log.error("<STR_LIT>"<EOL>% (str(kv), str(kv.type)))<EOL>continue<EOL><DEDENT><DEDENT>return config<EOL>

Converts Config protobuf message to python dictionary

        Values are converted according to the rules below:

        - Number string (e.g. "12" or "1.2") is appropriately converted to ``int`` or ``float``
        - Boolean string ("true", "True", "false" or "False") is converted to built-in boolean type
          (i.e. ``True`` or ``False``)
        - Normal string is inserted to dict as is
        - Serialized value is deserialized and inserted as a corresponding Python object


for i in range(len(topology.bolts)):<EOL><INDENT>for in_stream in topology.bolts[i].inputs:<EOL><INDENT>if in_stream.stream.component_name == self.my_component_name andin_stream.gtype == topology_pb2.Grouping.Value("<STR_LIT>"):<EOL><INDENT>if in_stream.type == topology_pb2.CustomGroupingObjectType.Value("<STR_LIT>"):<EOL><INDENT>custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object)<EOL>if isinstance(custom_grouping_obj, str):<EOL><INDENT>pex_loader.load_pex(self.topology_pex_abs_path)<EOL>grouping_cls =pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)<EOL>custom_grouping_obj = grouping_cls()<EOL><DEDENT>assert isinstance(custom_grouping_obj, ICustomGrouping)<EOL>self.custom_grouper.add(in_stream.stream.id,<EOL>self._get_taskids_for_component(topology.bolts[i].comp.name),<EOL>custom_grouping_obj,<EOL>self.my_component_name)<EOL><DEDENT>elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value("<STR_LIT>"):<EOL><INDENT>raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(in_stream.type))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Checks whether there are any bolts that consume any of my streams using custom grouping


self.custom_grouper.prepare(context)<EOL>

Prepares for custom grouping for this component

        :param context: Topology context


def choose_tasks_for_custom_grouping(self, stream_id, values):

return self.custom_grouper.choose_tasks(stream_id, values)<EOL>

Choose target task ids for custom grouping

        :return: task ids


def register_metrics(self, metrics_collector, interval):

for field, metrics in self.metrics.items():<EOL><INDENT>metrics_collector.register_metric(field, metrics, interval)<EOL><DEDENT>

Registers its metrics to a given metrics collector with a given interval


def update_count(self, name, incr_by=<NUM_LIT:1>, key=None):

if name not in self.metrics:<EOL><INDENT>Log.error("<STR_LIT>", name)<EOL><DEDENT>if key is None and isinstance(self.metrics[name], CountMetric):<EOL><INDENT>self.metrics[name].incr(incr_by)<EOL><DEDENT>elif key is not None and isinstance(self.metrics[name], MultiCountMetric):<EOL><INDENT>self.metrics[name].incr(key, incr_by)<EOL><DEDENT>else:<EOL><INDENT>Log.error("<STR_LIT>", name)<EOL><DEDENT>

Update the value of CountMetric or MultiCountMetric

        :type name: str
        :param name: name of the registered metric to be updated.
        :type incr_by: int
        :param incr_by: specifies how much to increment. Default is 1.
        :type key: str or None
        :param key: specifies a key for MultiCountMetric. Needs to be `None` for updating CountMetric.


def update_reduced_metric(self, name, value, key=None):

if name not in self.metrics:<EOL><INDENT>Log.error("<STR_LIT>", name)<EOL><DEDENT>if key is None and isinstance(self.metrics[name], ReducedMetric):<EOL><INDENT>self.metrics[name].update(value)<EOL><DEDENT>elif key is not None and isinstance(self.metrics[name], MultiReducedMetric):<EOL><INDENT>self.metrics[name].update(key, value)<EOL><DEDENT>else:<EOL><INDENT>Log.error("<STR_LIT>", name)<EOL><DEDENT>

Update the value of ReducedMetric or MultiReducedMetric

        :type name: str
        :param name: name of the registered metric to be updated.
        :param value: specifies a value to be reduced.
        :type key: str or None
        :param key: specifies a key for MultiReducedMetric. Needs to be `None` for updating
                    ReducedMetric.


self.update_count(self.RECEIVED_PKT_COUNT)<EOL>self.update_count(self.RECEIVED_PKT_SIZE, incr_by=received_pkt_size_bytes)<EOL>


self.update_count(self.SENT_PKT_COUNT)<EOL>self.update_count(self.SENT_PKT_SIZE, incr_by=sent_pkt_size_bytes)<EOL>


sys_config = system_config.get_sys_config()<EOL>interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])<EOL>collector = context.get_metrics_collector()<EOL>super(ComponentMetrics, self).register_metrics(collector, interval)<EOL>

Registers metrics to context

        :param context: Topology Context


self.update_count(self.OUT_QUEUE_FULL_COUNT)<EOL>

Apply update to the out-queue full count


self.update_count(self.EMIT_COUNT, key=stream_id)<EOL>


def serialize_data_tuple(self, stream_id, latency_in_ns):

self.update_count(self.TUPLE_SERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)<EOL>


to_init = [self.metrics[i] for i in self.to_multi_init<EOL>if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]<EOL>for out_stream in pplan_helper.get_my_spout().outputs:<EOL><INDENT>stream_id = out_stream.stream.id<EOL>for metric in to_init:<EOL><INDENT>metric.add_key(stream_id)<EOL><DEDENT><DEDENT>

Initializes the default values for a necessary set of MultiCountMetrics


self.update_reduced_metric(self.NEXT_TUPLE_LATENCY, latency_in_ns)<EOL>self.update_count(self.NEXT_TUPLE_COUNT)<EOL>


def acked_tuple(self, stream_id, complete_latency_ns):

self.update_count(self.ACK_COUNT, key=stream_id)<EOL>self.update_reduced_metric(self.COMPLETE_LATENCY, complete_latency_ns, key=stream_id)<EOL>


def failed_tuple(self, stream_id, fail_latency_ns):

self.update_count(self.FAIL_COUNT, key=stream_id)<EOL>self.update_reduced_metric(self.FAIL_LATENCY, fail_latency_ns, key=stream_id)<EOL>


self.update_reduced_metric(self.PENDING_ACKED_COUNT, count)<EOL>

Apply updates to the pending tuples count


self.update_count(self.TIMEOUT_COUNT, key=stream_id)<EOL>


<EOL>to_in_init = [self.metrics[i] for i in self.inputs_init<EOL>if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]<EOL>for in_stream in pplan_helper.get_my_bolt().inputs:<EOL><INDENT>stream_id = in_stream.stream.id<EOL>global_stream_id = in_stream.stream.component_name + "<STR_LIT:/>" + stream_id<EOL>for metric in to_in_init:<EOL><INDENT>metric.add_key(stream_id)<EOL>metric.add_key(global_stream_id)<EOL><DEDENT><DEDENT>to_out_init = [self.metrics[i] for i in self.outputs_init<EOL>if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]<EOL>for out_stream in pplan_helper.get_my_bolt().outputs:<EOL><INDENT>stream_id = out_stream.stream.id<EOL>for metric in to_out_init:<EOL><INDENT>metric.add_key(stream_id)<EOL><DEDENT><DEDENT>


def execute_tuple(self, stream_id, source_component, latency_in_ns):

self.update_count(self.EXEC_COUNT, key=stream_id)<EOL>self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, stream_id)<EOL>self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=stream_id)<EOL>global_stream_id = source_component + "<STR_LIT:/>" + stream_id<EOL>self.update_count(self.EXEC_COUNT, key=global_stream_id)<EOL>self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, global_stream_id)<EOL>self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=global_stream_id)<EOL>


def deserialize_data_tuple(self, stream_id, source_component, latency_in_ns):

self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)<EOL>global_stream_id = source_component + "<STR_LIT:/>" + stream_id<EOL>self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns,<EOL>key=global_stream_id)<EOL>

Apply updates to the deserialization metrics


def acked_tuple(self, stream_id, source_component, latency_in_ns):

self.update_count(self.ACK_COUNT, key=stream_id)<EOL>self.update_reduced_metric(self.PROCESS_LATENCY, latency_in_ns, stream_id)<EOL>global_stream_id = source_component + '<STR_LIT:/>' + stream_id<EOL>self.update_count(self.ACK_COUNT, key=global_stream_id)<EOL>self.update_reduced_metric(self.PROCESS_LATENCY, latency_in_ns, global_stream_id)<EOL>


def failed_tuple(self, stream_id, source_component, latency_in_ns):

self.update_count(self.FAIL_COUNT, key=stream_id)<EOL>self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, stream_id)<EOL>global_stream_id = source_component + '<STR_LIT:/>' + stream_id<EOL>self.update_count(self.FAIL_COUNT, key=global_stream_id)<EOL>self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, global_stream_id)<EOL>


def register_metric(self, name, metric, time_bucket_in_sec):

if name in self.metrics_map:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % name)<EOL><DEDENT>Log.debug("<STR_LIT>", name, str(time_bucket_in_sec))<EOL>self.metrics_map[name] = metric<EOL>if time_bucket_in_sec in self.time_bucket_in_sec_to_metrics_name:<EOL><INDENT>self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec].append(name)<EOL><DEDENT>else:<EOL><INDENT>self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec] = [name]<EOL>self._register_timer_task(time_bucket_in_sec)<EOL><DEDENT>

Registers a given metric

        :param name: name of the metric
        :param metric: IMetric object to be registered
        :param time_bucket_in_sec: time interval for update to the metrics manager


return self.task_id<EOL>

Property to get the task id of this component


return self.task_to_component_map.get(self.get_task_id())<EOL>

Property to get the component id of this component


return self.config<EOL>

Returns the cluster config for this component

        Note that the returned config is auto-typed map: <str -> any Python object>.


return str(self.topology.name)<EOL>


collector = self.get_metrics_collector()<EOL>collector.register_metric(name, metric, time_bucket_in_sec)<EOL>


<EOL>StreamId = namedtuple('<STR_LIT>', '<STR_LIT>')<EOL>if component_id in self.inputs:<EOL><INDENT>ret = {}<EOL>for istream in self.inputs.get(component_id):<EOL><INDENT>key = StreamId(id=istream.stream.id, component_name=istream.stream.component_name)<EOL>ret[key] = istream.gtype<EOL><DEDENT>return ret<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Returns the declared inputs to specified component

        :return: map <streamId namedtuple (same structure as protobuf msg) -> gtype>, or
                 None if not found


ret = []<EOL>for task_id, comp_id in self.task_to_component_map.items():<EOL><INDENT>if comp_id == component_id:<EOL><INDENT>ret.append(task_id)<EOL><DEDENT><DEDENT>return ret<EOL>

Returns the task ids allocated for the given component id


if not isinstance(task_hook, ITaskHook):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(task_hook)))<EOL><DEDENT>self.task_hooks.append(task_hook)<EOL>

Registers a specified task hook to this context

        :type task_hook: heron.instance.src.python.utils.topology.ITaskHook
        :param task_hook: Implementation of ITaskHook


return self.topology_pex_path<EOL>


if self.metrics_collector is None or not isinstance(self.metrics_collector, MetricsCollector):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>return self.metrics_collector<EOL>

Returns this context's metrics collector


for task_hook in self.task_hooks:<EOL><INDENT>task_hook.prepare(self.get_cluster_config(), self)<EOL><DEDENT>

invoke task hooks for after the spout/bolt's initialize() method


for task_hook in self.task_hooks:<EOL><INDENT>task_hook.clean_up()<EOL><DEDENT>

invoke task hooks for just before the spout/bolt's cleanup method


def invoke_hook_emit(self, values, stream_id, out_tasks):

if len(self.task_hooks) > <NUM_LIT:0>:<EOL><INDENT>emit_info = EmitInfo(values=values, stream_id=stream_id,<EOL>task_id=self.get_task_id(), out_tasks=out_tasks)<EOL>for task_hook in self.task_hooks:<EOL><INDENT>task_hook.emit(emit_info)<EOL><DEDENT><DEDENT>

invoke task hooks for every time a tuple is emitted in spout/bolt

        :type values: list
        :param values: values emitted
        :type stream_id: str
        :param stream_id: stream id into which tuple is emitted
        :type out_tasks: list
        :param out_tasks: list of custom grouping target task id


def invoke_hook_spout_ack(self, message_id, complete_latency_ns):

if len(self.task_hooks) > <NUM_LIT:0>:<EOL><INDENT>spout_ack_info = SpoutAckInfo(message_id=message_id,<EOL>spout_task_id=self.get_task_id(),<EOL>complete_latency_ms=complete_latency_ns *<EOL>system_constants.NS_TO_MS)<EOL>for task_hook in self.task_hooks:<EOL><INDENT>task_hook.spout_ack(spout_ack_info)<EOL><DEDENT><DEDENT>

invoke task hooks for every time spout acks a tuple

        :type message_id: str
        :param message_id: message id to which an acked tuple was anchored
        :type complete_latency_ns: float
        :param complete_latency_ns: complete latency in nano seconds


def invoke_hook_spout_fail(self, message_id, fail_latency_ns):

if len(self.task_hooks) > <NUM_LIT:0>:<EOL><INDENT>spout_fail_info = SpoutFailInfo(message_id=message_id,<EOL>spout_task_id=self.get_task_id(),<EOL>fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)<EOL>for task_hook in self.task_hooks:<EOL><INDENT>task_hook.spout_fail(spout_fail_info)<EOL><DEDENT><DEDENT>

invoke task hooks for every time spout fails a tuple

        :type message_id: str
        :param message_id: message id to which a failed tuple was anchored
        :type fail_latency_ns: float
        :param fail_latency_ns: fail latency in nano seconds


def invoke_hook_bolt_execute(self, heron_tuple, execute_latency_ns):

if len(self.task_hooks) > <NUM_LIT:0>:<EOL><INDENT>bolt_execute_info =BoltExecuteInfo(heron_tuple=heron_tuple,<EOL>executing_task_id=self.get_task_id(),<EOL>execute_latency_ms=execute_latency_ns * system_constants.NS_TO_MS)<EOL>for task_hook in self.task_hooks:<EOL><INDENT>task_hook.bolt_execute(bolt_execute_info)<EOL><DEDENT><DEDENT>

invoke task hooks for every time bolt processes a tuple

        :type heron_tuple: HeronTuple
        :param heron_tuple: tuple that is executed
        :type execute_latency_ns: float
        :param execute_latency_ns: execute latency in nano seconds


def invoke_hook_bolt_ack(self, heron_tuple, process_latency_ns):

if len(self.task_hooks) > <NUM_LIT:0>:<EOL><INDENT>bolt_ack_info = BoltAckInfo(heron_tuple=heron_tuple,<EOL>acking_task_id=self.get_task_id(),<EOL>process_latency_ms=process_latency_ns * system_constants.NS_TO_MS)<EOL>for task_hook in self.task_hooks:<EOL><INDENT>task_hook.bolt_ack(bolt_ack_info)<EOL><DEDENT><DEDENT>

invoke task hooks for every time bolt acks a tuple

        :type heron_tuple: HeronTuple
        :param heron_tuple: tuple that is acked
        :type process_latency_ns: float
        :param process_latency_ns: process latency in nano seconds


def invoke_hook_bolt_fail(self, heron_tuple, fail_latency_ns):

if len(self.task_hooks) > <NUM_LIT:0>:<EOL><INDENT>bolt_fail_info = BoltFailInfo(heron_tuple=heron_tuple,<EOL>failing_task_id=self.get_task_id(),<EOL>fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)<EOL>for task_hook in self.task_hooks:<EOL><INDENT>task_hook.bolt_fail(bolt_fail_info)<EOL><DEDENT><DEDENT>

invoke task hooks for every time bolt fails a tuple

        :type heron_tuple: HeronTuple
        :param heron_tuple: tuple that is failed
        :type fail_latency_ns: float
        :param fail_latency_ns: fail latency in nano seconds


if not config_path.endswith("<STR_LIT>"):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>with open(config_path, '<STR_LIT:r>') as f:<EOL><INDENT>config = yaml.load(f)<EOL><DEDENT>return config<EOL>

Reads yaml config file and returns auto-typed config_dict


if self.my_pplan_helper is None or self.my_instance is None:<EOL><INDENT>Log.error("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>hts = tuple_pb2.HeronTupleSet()<EOL>if hts2.HasField('<STR_LIT>'):<EOL><INDENT>hts.control.CopyFrom(hts2.control)<EOL><DEDENT>else:<EOL><INDENT>hdts = tuple_pb2.HeronDataTupleSet()<EOL>hdts.stream.CopyFrom(hts2.data.stream)<EOL>try:<EOL><INDENT>for trunk in hts2.data.tuples:<EOL><INDENT>added_tuple = hdts.tuples.add()<EOL>added_tuple.ParseFromString(trunk)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>Log.exception('<STR_LIT>')<EOL><DEDENT>hts.data.CopyFrom(hdts)<EOL><DEDENT>self.in_stream.offer(hts)<EOL>if self.my_pplan_helper.is_topology_running():<EOL><INDENT>self.my_instance.py_class.process_incoming_tuples()<EOL><DEDENT><DEDENT>

Called when new HeronTupleSet2 arrives
           Convert(Assemble) HeronTupleSet2(raw byte array) to HeronTupleSet
           See more at GitHub PR #1421
        :param tuple_msg_set: HeronTupleSet2 type


self.in_stream.offer(ckptmsg)<EOL>if self.my_pplan_helper.is_topology_running():<EOL><INDENT>self.my_instance.py_class.process_incoming_tuples()<EOL><DEDENT>

Called when we get InitiateStatefulCheckpoint message
        :param ckptmsg: InitiateStatefulCheckpoint type


Log.info("<STR_LIT>" % start_msg.checkpoint_id)<EOL>self.is_stateful_started = True<EOL>self.start_instance_if_possible()<EOL>

Called when we receive StartInstanceStatefulProcessing message
        :param start_msg: StartInstanceStatefulProcessing type


Log.info("<STR_LIT>" % restore_msg.state.checkpoint_id)<EOL>if self.is_stateful_started:<EOL><INDENT>self.my_instance.py_class.stop()<EOL>self.my_instance.py_class.clear_collector()<EOL>self.is_stateful_started = False<EOL><DEDENT>self.in_stream.clear()<EOL>self.out_stream.clear()<EOL>if self.stateful_state is not None:<EOL><INDENT>self.stateful_state.clear()<EOL><DEDENT>if restore_msg.state.state is not None and restore_msg.state.state:<EOL><INDENT>try:<EOL><INDENT>self.stateful_state = self.serializer.deserialize(restore_msg.state.state)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise RuntimeError("<STR_LIT>" + str(e))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>Log.info("<STR_LIT>")<EOL><DEDENT>if self.stateful_state is None:<EOL><INDENT>self.stateful_state = HashMapState()<EOL><DEDENT>Log.info("<STR_LIT>")<EOL>resp = ckptmgr_pb2.RestoreInstanceStateResponse()<EOL>resp.status.status = common_pb2.StatusCode.Value("<STR_LIT:OK>")<EOL>resp.checkpoint_id = restore_msg.state.checkpoint_id<EOL>self._stmgr_client.send_message(resp)<EOL>

Called when we receive RestoreInstanceStateRequest message
        :param restore_msg: RestoreInstanceStateRequest type


while not self.out_stream.is_empty() and self._stmgr_client.is_registered:<EOL><INDENT>tuple_set = self.out_stream.poll()<EOL>if isinstance(tuple_set, tuple_pb2.HeronTupleSet):<EOL><INDENT>tuple_set.src_task_id = self.my_pplan_helper.my_task_id<EOL>self.gateway_metrics.update_sent_packet(tuple_set.ByteSize())<EOL><DEDENT>self._stmgr_client.send_message(tuple_set)<EOL><DEDENT>

Send messages in out_stream to the Stream Manager


assert self.my_pplan_helper is not None<EOL>assert self.my_instance is not None and self.my_instance.py_class is not None<EOL>if self.my_pplan_helper.get_topology_state() != new_helper.get_topology_state():<EOL><INDENT>self.my_pplan_helper = new_helper<EOL>if new_helper.is_topology_running():<EOL><INDENT>if not self.is_instance_started:<EOL><INDENT>self.start_instance_if_possible()<EOL><DEDENT>self.my_instance.py_class.invoke_activate()<EOL><DEDENT>elif new_helper.is_topology_paused():<EOL><INDENT>self.my_instance.py_class.invoke_deactivate()<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % new_helper.get_topology_state())<EOL><DEDENT><DEDENT>else:<EOL><INDENT>Log.info("<STR_LIT>")<EOL><DEDENT>

Called when state change is commanded by stream manager


new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id,<EOL>self.topo_pex_file_abs_path)<EOL>if self.my_pplan_helper is not None and(self.my_pplan_helper.my_component_name != new_helper.my_component_name or<EOL>self.my_pplan_helper.my_task_id != new_helper.my_task_id):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>new_helper.set_topology_context(self.metrics_collector)<EOL>if self.my_pplan_helper is None:<EOL><INDENT>Log.info("<STR_LIT>")<EOL>Log.info("<STR_LIT>")<EOL>self._handle_assignment_msg(new_helper)<EOL><DEDENT>else:<EOL><INDENT>Log.info("<STR_LIT>")<EOL>Log.info("<STR_LIT>",<EOL>self.my_pplan_helper.get_topology_state(), new_helper.get_topology_state())<EOL>self._handle_state_change_msg(new_helper)<EOL><DEDENT>

Called when new NewInstanceAssignmentMessage arrives

        Tells this instance to become either spout/bolt.

        :param pplan: PhysicalPlan proto


<EOL>tmp_file = get_tmp_filename()<EOL>with open(tmp_file, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(content)<EOL>f.flush()<EOL>os.fsync(f.fileno())<EOL><DEDENT>os.rename(tmp_file, path)<EOL>

file.write(...) is not atomic.
We write to a tmp file and then rename to target path since rename is atomic.
We do this to avoid the content of file is dirty read/partially read by others.


<EOL>return lambda line: Log.info("<STR_LIT>", cmd, line.rstrip('<STR_LIT:\n>'))<EOL>

Simple function callback that is used to log the streaming output of a subprocess command
    :param cmd: the name of the command which will be added to the log line
    :return: None


<EOL>def signal_handler(signal_to_handle, frame):<EOL><INDENT>Log.info('<STR_LIT>', signal_to_handle)<EOL>executor.stop_state_manager_watches()<EOL>sys.exit(signal_to_handle)<EOL><DEDENT>def cleanup():<EOL><INDENT>"""<STR_LIT>"""<EOL>Log.info('<STR_LIT>')<EOL>for pid in list(executor.processes_to_monitor.keys()):<EOL><INDENT>os.kill(pid, signal.SIGTERM)<EOL><DEDENT>time.sleep(<NUM_LIT:5>)<EOL>os.killpg(<NUM_LIT:0>, signal.SIGTERM)<EOL><DEDENT>shardid = executor.shard<EOL>log.configure(logfile='<STR_LIT>' % shardid)<EOL>pid = os.getpid()<EOL>sid = os.getsid(pid)<EOL>if pid != sid:<EOL><INDENT>Log.info('<STR_LIT>')<EOL>os.setpgrp() <EOL><DEDENT>Log.info('<STR_LIT>')<EOL>signal.signal(signal.SIGTERM, signal_handler)<EOL>Log.info('<STR_LIT>')<EOL>atexit.register(cleanup)<EOL>


setup(executor)<EOL>executor.start_state_manager_watches()<EOL>executor.start_process_monitor()<EOL>


<EOL>shell_env = os.environ.copy()<EOL>shell_env["<STR_LIT>"] = os.path.join(os.path.abspath('<STR_LIT:.>'), "<STR_LIT>")<EOL>executor = HeronExecutor(sys.argv, shell_env)<EOL>executor.initialize()<EOL>start(executor)<EOL>

Register exit handlers, initialize the executor and run it.


def __init__(self, process, name, command, attempts=<NUM_LIT:1>):

self.process = process<EOL>self.pid = process.pid<EOL>self.name = name<EOL>self.command = command<EOL>self.command_str = command.__str__() <EOL>self.attempts = attempts<EOL>

Container for info related to a running process
:param process: the process POpen object
:param name: the logical (i.e., unique) name of the process
:param command: an array of strings comprising the command and it's args
:param attempts: how many times the command has been run (defaults to 1)


self.shard = parsed_args.shard<EOL>self.topology_name = parsed_args.topology_name<EOL>self.topology_id = parsed_args.topology_id<EOL>self.topology_defn_file = parsed_args.topology_defn_file<EOL>self.state_manager_connection = parsed_args.state_manager_connection<EOL>self.state_manager_root = parsed_args.state_manager_root<EOL>self.state_manager_config_file = parsed_args.state_manager_config_file<EOL>self.tmaster_binary = parsed_args.tmaster_binary<EOL>self.stmgr_binary = parsed_args.stmgr_binary<EOL>self.metrics_manager_classpath = parsed_args.metrics_manager_classpath<EOL>self.metricscache_manager_classpath = parsed_args.metricscache_manager_classpath<EOL>self.instance_jvm_opts =base64.b64decode(parsed_args.instance_jvm_opts.lstrip('<STR_LIT:">').<EOL>rstrip('<STR_LIT:">').replace('<STR_LIT>', '<STR_LIT:=>').replace('<STR_LIT>', '<STR_LIT:=>'))<EOL>self.classpath = parsed_args.classpath<EOL>if is_docker_environment():<EOL><INDENT>self.master_host = os.environ.get('<STR_LIT>') if '<STR_LIT>' in os.environ else socket.gethostname()<EOL><DEDENT>else:<EOL><INDENT>self.master_host = socket.gethostname()<EOL><DEDENT>self.master_port = parsed_args.master_port<EOL>self.tmaster_controller_port = parsed_args.tmaster_controller_port<EOL>self.tmaster_stats_port = parsed_args.tmaster_stats_port<EOL>self.heron_internals_config_file = parsed_args.heron_internals_config_file<EOL>self.override_config_file = parsed_args.override_config_file<EOL>self.component_ram_map =[{x.split('<STR_LIT::>')[<NUM_LIT:0>]:<EOL>int(x.split('<STR_LIT::>')[<NUM_LIT:1>])} for x in parsed_args.component_ram_map.split('<STR_LIT:U+002C>')]<EOL>self.component_ram_map =functools.reduce(lambda x, y: dict(list(x.items()) + list(y.items())), self.component_ram_map)<EOL>self.component_jvm_opts = {}<EOL>component_jvm_opts_in_json =base64.b64decode(parsed_args.component_jvm_opts.<EOL>lstrip('<STR_LIT:">').rstrip('<STR_LIT:">').replace('<STR_LIT>', '<STR_LIT:=>').replace('<STR_LIT>', '<STR_LIT:=>'))<EOL>if component_jvm_opts_in_json != "<STR_LIT>":<EOL><INDENT>for (k, v) in list(json.loads(component_jvm_opts_in_json).items()):<EOL><INDENT>self.component_jvm_opts[base64.b64decode(k)] = base64.b64decode(v)<EOL><DEDENT><DEDENT>self.pkg_type = parsed_args.pkg_type<EOL>self.topology_binary_file = parsed_args.topology_binary_file<EOL>self.heron_java_home = parsed_args.heron_java_home<EOL>self.shell_port = parsed_args.shell_port<EOL>self.heron_shell_binary = parsed_args.heron_shell_binary<EOL>self.metrics_manager_port = parsed_args.metrics_manager_port<EOL>self.metricscache_manager_master_port = parsed_args.metricscache_manager_master_port<EOL>self.metricscache_manager_stats_port = parsed_args.metricscache_manager_stats_port<EOL>self.cluster = parsed_args.cluster<EOL>self.role = parsed_args.role<EOL>self.environment = parsed_args.environment<EOL>self.instance_classpath = parsed_args.instance_classpath<EOL>self.metrics_sinks_config_file = parsed_args.metrics_sinks_config_file<EOL>self.scheduler_classpath = parsed_args.scheduler_classpath<EOL>self.scheduler_port = parsed_args.scheduler_port<EOL>self.python_instance_binary = parsed_args.python_instance_binary<EOL>self.cpp_instance_binary = parsed_args.cpp_instance_binary<EOL>self.is_stateful_topology = (parsed_args.is_stateful.lower() == '<STR_LIT:true>')<EOL>self.checkpoint_manager_classpath = parsed_args.checkpoint_manager_classpath<EOL>self.checkpoint_manager_port = parsed_args.checkpoint_manager_port<EOL>self.checkpoint_manager_ram = parsed_args.checkpoint_manager_ram<EOL>self.stateful_config_file = parsed_args.stateful_config_file<EOL>self.metricscache_manager_mode = parsed_args.metricscache_manager_modeif parsed_args.metricscache_manager_mode else "<STR_LIT>"<EOL>self.health_manager_mode = parsed_args.health_manager_mode<EOL>self.health_manager_classpath = '<STR_LIT>'% (self.scheduler_classpath, parsed_args.health_manager_classpath)<EOL>self.jvm_remote_debugger_ports =parsed_args.jvm_remote_debugger_ports.split("<STR_LIT:U+002C>")if parsed_args.jvm_remote_debugger_ports else None<EOL>


Log.info("<STR_LIT>" % args)<EOL>parser = argparse.ArgumentParser()<EOL>parser.add_argument("<STR_LIT>", type=int, required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=False)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", type=int, required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=True)<EOL>parser.add_argument("<STR_LIT>", required=False,<EOL>help="<STR_LIT>")<EOL>parsed_args, unknown_args = parser.parse_known_args(args[<NUM_LIT:1>:])<EOL>if unknown_args:<EOL><INDENT>Log.error('<STR_LIT>' % unknown_args[<NUM_LIT:0>])<EOL>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return parsed_args<EOL>

Uses python argparse to collect positional args


create_folders = Command('<STR_LIT>' % self.log_dir, self.shell_env)<EOL>self.run_command_or_exit(create_folders)<EOL>chmod_logs_dir = Command('<STR_LIT>' % self.log_dir, self.shell_env)<EOL>self.run_command_or_exit(chmod_logs_dir)<EOL>chmod_x_binaries = [self.tmaster_binary, self.stmgr_binary, self.heron_shell_binary]<EOL>for binary in chmod_x_binaries:<EOL><INDENT>stat_result = os.stat(binary)[stat.ST_MODE]<EOL>if not stat_result & stat.S_IXOTH:<EOL><INDENT>chmod_binary = Command('<STR_LIT>' % binary, self.shell_env)<EOL>self.run_command_or_exit(chmod_binary)<EOL><DEDENT><DEDENT>log_pid_for_process(get_heron_executor_process_name(self.shard), os.getpid())<EOL>

Initialize the environment. Done with a method call outside of the constructor for 2 reasons:
1. Unit tests probably won't want/need to do this
2. We don't initialize the logger (also something unit tests don't want) until after the
constructor


def _get_metricsmgr_cmd(self, metricsManagerId, sink_config_file, port):

metricsmgr_main_class = '<STR_LIT>'<EOL>metricsmgr_cmd = [os.path.join(self.heron_java_home, '<STR_LIT>'),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>self.metrics_manager_classpath,<EOL>metricsmgr_main_class,<EOL>'<STR_LIT>' + metricsManagerId,<EOL>'<STR_LIT>' + str(port),<EOL>'<STR_LIT>' + self.topology_name,<EOL>'<STR_LIT>' + self.cluster,<EOL>'<STR_LIT>' + self.role,<EOL>'<STR_LIT>' + self.environment,<EOL>'<STR_LIT>' + self.topology_id,<EOL>'<STR_LIT>' + self.heron_internals_config_file,<EOL>'<STR_LIT>' + self.override_config_file,<EOL>'<STR_LIT>' + sink_config_file]<EOL>return Command(metricsmgr_cmd, self.shell_env)<EOL>

get the command to start the metrics manager processes


metricscachemgr_main_class = '<STR_LIT>'<EOL>metricscachemgr_cmd = [os.path.join(self.heron_java_home, '<STR_LIT>'),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>self.metricscache_manager_classpath,<EOL>metricscachemgr_main_class,<EOL>"<STR_LIT>", '<STR_LIT>',<EOL>"<STR_LIT>", self.metricscache_manager_master_port,<EOL>"<STR_LIT>", self.metricscache_manager_stats_port,<EOL>"<STR_LIT>", self.topology_name,<EOL>"<STR_LIT>", self.topology_id,<EOL>"<STR_LIT>", self.heron_internals_config_file,<EOL>"<STR_LIT>", self.override_config_file,<EOL>"<STR_LIT>", self.metrics_sinks_config_file,<EOL>"<STR_LIT>", self.cluster,<EOL>"<STR_LIT>", self.role,<EOL>"<STR_LIT>", self.environment]<EOL>return Command(metricscachemgr_cmd, self.shell_env)<EOL>


healthmgr_main_class = '<STR_LIT>'<EOL>healthmgr_cmd = [os.path.join(self.heron_java_home, '<STR_LIT>'),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', self.health_manager_classpath,<EOL>healthmgr_main_class,<EOL>"<STR_LIT>", self.cluster,<EOL>"<STR_LIT>", self.role,<EOL>"<STR_LIT>", self.environment,<EOL>"<STR_LIT>", self.topology_name,<EOL>"<STR_LIT>", self.metrics_manager_port]<EOL>return Command(healthmgr_cmd, self.shell_env)<EOL>

get the command to start the topology health manager processes


retval = {}<EOL>tmaster_cmd_lst = [<EOL>self.tmaster_binary,<EOL>'<STR_LIT>' % self.topology_name,<EOL>'<STR_LIT>' % self.topology_id,<EOL>'<STR_LIT>' % self.state_manager_connection,<EOL>'<STR_LIT>' % self.state_manager_root,<EOL>'<STR_LIT>' % self.master_host,<EOL>'<STR_LIT>' % str(self.master_port),<EOL>'<STR_LIT>' % str(self.tmaster_controller_port),<EOL>'<STR_LIT>' % str(self.tmaster_stats_port),<EOL>'<STR_LIT>' % self.heron_internals_config_file,<EOL>'<STR_LIT>' % self.override_config_file,<EOL>'<STR_LIT>' % self.metrics_sinks_config_file,<EOL>'<STR_LIT>' % str(self.metrics_manager_port),<EOL>'<STR_LIT>' % str(self.checkpoint_manager_port)]<EOL>tmaster_env = self.shell_env.copy() if self.shell_env is not None else {}<EOL>tmaster_cmd = Command(tmaster_cmd_lst, tmaster_env)<EOL>if os.environ.get('<STR_LIT>') is not None:<EOL><INDENT>tmaster_cmd.env.update({<EOL>'<STR_LIT>': "<STR_LIT>",<EOL>'<STR_LIT>': "<STR_LIT>"<EOL>})<EOL><DEDENT>retval["<STR_LIT>"] = tmaster_cmd<EOL>if self.metricscache_manager_mode.lower() != "<STR_LIT>":<EOL><INDENT>retval["<STR_LIT>"] = self._get_metrics_cache_cmd()<EOL><DEDENT>if self.health_manager_mode.lower() != "<STR_LIT>":<EOL><INDENT>retval["<STR_LIT>"] = self._get_healthmgr_cmd()<EOL><DEDENT>retval[self.metricsmgr_ids[<NUM_LIT:0>]] = self._get_metricsmgr_cmd(<EOL>self.metricsmgr_ids[<NUM_LIT:0>],<EOL>self.metrics_sinks_config_file,<EOL>self.metrics_manager_port)<EOL>if self.is_stateful_topology:<EOL><INDENT>retval.update(self._get_ckptmgr_process())<EOL><DEDENT>return retval<EOL>

get the command to start the tmaster processes


retval = {}<EOL>instance_plans = self._get_instance_plans(self.packing_plan, self.shard)<EOL>instance_info = []<EOL>for instance_plan in instance_plans:<EOL><INDENT>global_task_id = instance_plan.task_id<EOL>component_index = instance_plan.component_index<EOL>component_name = instance_plan.component_name<EOL>instance_id = "<STR_LIT>" % (str(self.shard), component_name, global_task_id)<EOL>instance_info.append((instance_id, component_name, global_task_id, component_index))<EOL><DEDENT>stmgr_cmd_lst = [<EOL>self.stmgr_binary,<EOL>'<STR_LIT>' % self.topology_name,<EOL>'<STR_LIT>' % self.topology_id,<EOL>'<STR_LIT>' % self.topology_defn_file,<EOL>'<STR_LIT>' % self.state_manager_connection,<EOL>'<STR_LIT>' % self.state_manager_root,<EOL>'<STR_LIT>' % self.stmgr_ids[self.shard],<EOL>'<STR_LIT>' % '<STR_LIT:U+002C>'.join([x[<NUM_LIT:0>] for x in instance_info]),<EOL>'<STR_LIT>' % self.master_host,<EOL>'<STR_LIT>' % str(self.master_port),<EOL>'<STR_LIT>' % str(self.tmaster_controller_port),<EOL>'<STR_LIT>' % str(self.metrics_manager_port),<EOL>'<STR_LIT>' % str(self.shell_port),<EOL>'<STR_LIT>' % self.heron_internals_config_file,<EOL>'<STR_LIT>' % self.override_config_file,<EOL>'<STR_LIT>' % str(self.checkpoint_manager_port),<EOL>'<STR_LIT>' % self.ckptmgr_ids[self.shard],<EOL>'<STR_LIT>' % self.metricscache_manager_mode.lower()]<EOL>stmgr_env = self.shell_env.copy() if self.shell_env is not None else {}<EOL>stmgr_cmd = Command(stmgr_cmd_lst, stmgr_env)<EOL>if os.environ.get('<STR_LIT>') is not None:<EOL><INDENT>stmgr_cmd.env.update({<EOL>'<STR_LIT>': "<STR_LIT>",<EOL>'<STR_LIT>': "<STR_LIT>"<EOL>})<EOL><DEDENT>retval[self.stmgr_ids[self.shard]] = stmgr_cmd<EOL>retval[self.metricsmgr_ids[self.shard]] = self._get_metricsmgr_cmd(<EOL>self.metricsmgr_ids[self.shard],<EOL>self.metrics_sinks_config_file,<EOL>self.metrics_manager_port<EOL>)<EOL>if self.is_stateful_topology:<EOL><INDENT>retval.update(self._get_ckptmgr_process())<EOL><DEDENT>if self.pkg_type == '<STR_LIT>' or self.pkg_type == '<STR_LIT>':<EOL><INDENT>retval.update(self._get_java_instance_cmd(instance_info))<EOL><DEDENT>elif self.pkg_type == '<STR_LIT>':<EOL><INDENT>retval.update(self._get_python_instance_cmd(instance_info))<EOL><DEDENT>elif self.pkg_type == '<STR_LIT>':<EOL><INDENT>retval.update(self._get_cpp_instance_cmd(instance_info))<EOL><DEDENT>elif self.pkg_type == '<STR_LIT>':<EOL><INDENT>retval.update(self._get_cpp_instance_cmd(instance_info))<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % self.pkg_type)<EOL><DEDENT>return retval<EOL>

Returns the processes to handle streams, including the stream-mgr and the user code containing
the stream logic of the topology


ckptmgr_main_class = '<STR_LIT>'<EOL>ckptmgr_ram_mb = self.checkpoint_manager_ram / (<NUM_LIT> * <NUM_LIT>)<EOL>ckptmgr_cmd = [os.path.join(self.heron_java_home, "<STR_LIT>"),<EOL>'<STR_LIT>' % ckptmgr_ram_mb,<EOL>'<STR_LIT>' % ckptmgr_ram_mb,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>self.checkpoint_manager_classpath,<EOL>ckptmgr_main_class,<EOL>'<STR_LIT>' + self.topology_name,<EOL>'<STR_LIT>' + self.topology_id,<EOL>'<STR_LIT:-c>' + self.ckptmgr_ids[self.shard],<EOL>'<STR_LIT>' + self.checkpoint_manager_port,<EOL>'<STR_LIT>' + self.stateful_config_file,<EOL>'<STR_LIT>' + self.override_config_file,<EOL>'<STR_LIT>' + self.heron_internals_config_file]<EOL>retval = {}<EOL>retval[self.ckptmgr_ids[self.shard]] = Command(ckptmgr_cmd, self.shell_env)<EOL>return retval<EOL>

Get the command to start the checkpoint manager process


def _get_instance_plans(self, packing_plan, container_id):

this_container_plan = None<EOL>for container_plan in packing_plan.container_plans:<EOL><INDENT>if container_plan.id == container_id:<EOL><INDENT>this_container_plan = container_plan<EOL><DEDENT><DEDENT>if this_container_plan is None:<EOL><INDENT>return None<EOL><DEDENT>return this_container_plan.instance_plans<EOL>

For the given packing_plan, return the container plan with the given container_id. If protobufs
supported maps, we could just get the plan by id, but it doesn't so we have a collection of
containers to iterate over.


retval = {}<EOL>retval[self.heron_shell_ids[self.shard]] = Command([<EOL>'<STR_LIT:%s>' % self.heron_shell_binary,<EOL>'<STR_LIT>' % self.shell_port,<EOL>'<STR_LIT>' % (self.log_dir, self.shard),<EOL>'<STR_LIT>' % self.topology_id], self.shell_env)<EOL>return retval<EOL>

Get a map from all daemon services' name to the command to start them


def _wait_process_std_out_err(self, name, process):

proc.stream_process_stdout(process, stdout_log_fn(name))<EOL>process.wait()<EOL>

Wait for the termination of a process and log its stdout & stderr


Log.info("<STR_LIT>")<EOL>processes_to_monitor = {}<EOL>for (name, command) in list(commands.items()):<EOL><INDENT>p = self._run_process(name, command)<EOL>processes_to_monitor[p.pid] = ProcessInfo(p, name, command)<EOL>log_pid_for_process(name, p.pid)<EOL><DEDENT>with self.process_lock:<EOL><INDENT>self.processes_to_monitor.update(processes_to_monitor)<EOL><DEDENT>

Start all commands and add them to the dict of processes to be monitored


<EOL>Log.info("<STR_LIT>")<EOL>while True:<EOL><INDENT>if len(self.processes_to_monitor) > <NUM_LIT:0>:<EOL><INDENT>(pid, status) = os.wait()<EOL>with self.process_lock:<EOL><INDENT>if pid in list(self.processes_to_monitor.keys()):<EOL><INDENT>old_process_info = self.processes_to_monitor[pid]<EOL>name = old_process_info.name<EOL>command = old_process_info.command<EOL>Log.info("<STR_LIT>" % (name, pid, status, command))<EOL>self._wait_process_std_out_err(name, old_process_info.process)<EOL>if os.path.isfile("<STR_LIT>" % pid):<EOL><INDENT>os.system("<STR_LIT>" % pid)<EOL><DEDENT>if old_process_info.attempts >= self.max_runs:<EOL><INDENT>Log.info("<STR_LIT>" % name)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>time.sleep(self.interval_between_runs)<EOL>p = self._run_process(name, command)<EOL>del self.processes_to_monitor[pid]<EOL>self.processes_to_monitor[p.pid] =ProcessInfo(p, name, command, old_process_info.attempts + <NUM_LIT:1>)<EOL>log_pid_for_process(name, p.pid)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Monitor all processes in processes_to_monitor dict,
        restarting any if they fail, up to max_runs times.


<EOL>if len(self.packing_plan.container_plans) == <NUM_LIT:0>:<EOL><INDENT>return {}<EOL><DEDENT>if self._get_instance_plans(self.packing_plan, self.shard) is None and self.shard != <NUM_LIT:0>:<EOL><INDENT>retval = {}<EOL>retval['<STR_LIT>'] = Command([<EOL>'<STR_LIT:%s>' % self.heron_shell_binary,<EOL>'<STR_LIT>' % self.shell_port,<EOL>'<STR_LIT>' % (self.log_dir, self.shard),<EOL>'<STR_LIT>' % self.topology_id], self.shell_env)<EOL>return retval<EOL><DEDENT>if self.shard == <NUM_LIT:0>:<EOL><INDENT>commands = self._get_tmaster_processes()<EOL><DEDENT>else:<EOL><INDENT>self._untar_if_needed()<EOL>commands = self._get_streaming_processes()<EOL><DEDENT>commands.update(self._get_heron_support_processes())<EOL>return commands<EOL>

Prepare either TMaster or Streaming commands according to shard.
The Shell command is attached to all containers. The empty container plan and non-exist
container plan are bypassed.


def get_command_changes(self, current_commands, updated_commands):

commands_to_kill = {}<EOL>commands_to_keep = {}<EOL>commands_to_start = {}<EOL>for current_name, current_command in list(current_commands.items()):<EOL><INDENT>if current_name in list(updated_commands.keys()) andcurrent_command == updated_commands[current_name] andnot current_name.startswith('<STR_LIT>'):<EOL><INDENT>commands_to_keep[current_name] = current_command<EOL><DEDENT>else:<EOL><INDENT>commands_to_kill[current_name] = current_command<EOL><DEDENT><DEDENT>for updated_name, updated_command in list(updated_commands.items()):<EOL><INDENT>if updated_name not in list(commands_to_keep.keys()):<EOL><INDENT>commands_to_start[updated_name] = updated_command<EOL><DEDENT><DEDENT>return commands_to_kill, commands_to_keep, commands_to_start<EOL>

Compares the current command with updated command to return a 3-tuple of dicts,
keyed by command name: commands_to_kill, commands_to_keep and commands_to_start.


with self.process_lock:<EOL><INDENT>current_commands = dict(list(map((lambda process: (process.name, process.command)),<EOL>list(self.processes_to_monitor.values()))))<EOL>updated_commands = self.get_commands_to_run()<EOL>commands_to_kill, commands_to_keep, commands_to_start =self.get_command_changes(current_commands, updated_commands)<EOL>Log.info("<STR_LIT>" % sorted(current_commands.keys()))<EOL>Log.info("<STR_LIT>" % sorted(updated_commands.keys()))<EOL>Log.info("<STR_LIT>" % sorted(commands_to_kill.keys()))<EOL>Log.info("<STR_LIT>" % sorted(commands_to_keep.keys()))<EOL>Log.info("<STR_LIT>" % sorted(commands_to_start.keys()))<EOL>self._kill_processes(commands_to_kill)<EOL>self._start_processes(commands_to_start)<EOL>Log.info("<STR_LIT>" %<EOL>(len(commands_to_kill), len(commands_to_keep),<EOL>len(commands_to_start), len(self.processes_to_monitor)))<EOL><DEDENT>

Determines the commands to be run and compares them with the existing running commands.
        Then starts new ones required and kills old ones no longer required.


Log.info("<STR_LIT>")<EOL>statemgr_config = StateMgrConfig()<EOL>statemgr_config.set_state_locations(configloader.load_state_manager_locations(<EOL>self.cluster, state_manager_config_file=self.state_manager_config_file,<EOL>overrides={"<STR_LIT>": self.state_manager_connection}))<EOL>try:<EOL><INDENT>self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)<EOL>for state_manager in self.state_managers:<EOL><INDENT>state_manager.start()<EOL><DEDENT><DEDENT>except Exception as ex:<EOL><INDENT>Log.error("<STR_LIT>" % ex)<EOL>traceback.print_exc()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>def on_packing_plan_watch(state_manager, new_packing_plan):<EOL><INDENT>Log.debug("<STR_LIT>" %<EOL>(self.shard, str(self.packing_plan), str(new_packing_plan)))<EOL>if self.packing_plan != new_packing_plan:<EOL><INDENT>Log.info("<STR_LIT>"<EOL>% self.shard)<EOL>self.update_packing_plan(new_packing_plan)<EOL>Log.info("<STR_LIT>")<EOL>self.launch()<EOL><DEDENT>else:<EOL><INDENT>Log.info(<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>for state_manager in self.state_managers:<EOL><INDENT>onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager)<EOL>state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)<EOL>Log.info("<STR_LIT>" %<EOL>str(state_manager))<EOL><DEDENT>

Receive updates to the packing plan from the statemgrs and update processes as needed.


pex = zipfile.ZipFile(abs_path_to_pex, mode='<STR_LIT:r>')<EOL>deps = list(set([re.match(egg_regex, i).group(<NUM_LIT:1>) for i in pex.namelist()<EOL>if re.match(egg_regex, i) is not None]))<EOL>return deps<EOL>

Get a list of paths to included dependencies in the specified pex file

    Note that dependencies are located under `.deps` directory


abs_path_to_pex = os.path.abspath(path_to_pex)<EOL>Log.debug("<STR_LIT>" % abs_path_to_pex)<EOL>if abs_path_to_pex not in sys.path:<EOL><INDENT>sys.path.insert(<NUM_LIT:0>, os.path.dirname(abs_path_to_pex))<EOL><DEDENT>if include_deps:<EOL><INDENT>for dep in _get_deps_list(abs_path_to_pex):<EOL><INDENT>to_join = os.path.join(os.path.dirname(abs_path_to_pex), dep)<EOL>if to_join not in sys.path:<EOL><INDENT>Log.debug("<STR_LIT>" % dep)<EOL>sys.path.insert(<NUM_LIT:0>, to_join)<EOL><DEDENT><DEDENT><DEDENT>Log.debug("<STR_LIT>" % str(sys.path))<EOL>

Loads pex file and its dependencies to the current python path


<EOL>importer = zipimport.zipimporter(abs_pex_path)<EOL>importer.load_module("<STR_LIT>")<EOL>to_load_lst = class_path.split('<STR_LIT:.>')[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>loaded = ['<STR_LIT>']<EOL>loaded_mod = None<EOL>for to_load in to_load_lst:<EOL><INDENT>sub_importer = zipimport.zipimporter(os.path.join(abs_pex_path, '<STR_LIT:/>'.join(loaded)))<EOL>loaded_mod = sub_importer.load_module(to_load)<EOL>loaded.append(to_load)<EOL><DEDENT>return loaded_mod<EOL>

Resolves duplicate package suffix problems

    When dynamically loading a pex file and a corresponding python class (bolt/spout/topology),
    if the top level package in which to-be-loaded classes reside is named 'heron', the path conflicts
    with this Heron Instance pex package (heron.instance.src.python...), making the Python
    interpreter unable to find the target class in a given pex file.
    This function resolves this issue by individually loading packages with suffix `heron` to
    avoid this issue.

    However, if a dependent module/class that is not directly specified under ``class_path``
    and has conflicts with other native heron packages, there is a possibility that
    such a class/module might not be imported correctly. For example, if a given ``class_path`` was
    ``heron.common.src.module.Class``, but it has a dependent module (such as by import statement),
    ``heron.common.src.python.dep_module.DepClass`` for example, pex_loader does not guarantee that
    ``DepClass` is imported correctly. This is because ``heron.common.src.python.dep_module`` is not
    explicitly added to sys.path, while ``heron.common.src.python`` module exists as the native heron
    package, from which ``dep_module`` cannot be found, so Python interpreter may raise ImportError.

    The best way to avoid this issue is NOT to dynamically load a pex file whose top level package
    name is ``heron``. Note that this method is included because some of the example topologies and
    tests have to have a pex with its top level package name of ``heron``.


abs_path_to_pex = os.path.abspath(path_to_pex)<EOL>Log.debug("<STR_LIT>" % abs_path_to_pex)<EOL>Log.debug("<STR_LIT>" % python_class_name)<EOL>split = python_class_name.split('<STR_LIT:.>')<EOL>from_path = '<STR_LIT:.>'.join(split[:-<NUM_LIT:1>])<EOL>import_name = python_class_name.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>Log.debug("<STR_LIT>" % (from_path, import_name))<EOL>if python_class_name.startswith("<STR_LIT>"):<EOL><INDENT>try:<EOL><INDENT>mod = resolve_heron_suffix_issue(abs_path_to_pex, python_class_name)<EOL>return getattr(mod, import_name)<EOL><DEDENT>except:<EOL><INDENT>Log.error("<STR_LIT>" % python_class_name)<EOL><DEDENT><DEDENT>mod = __import__(from_path, fromlist=[import_name], level=-<NUM_LIT:1>)<EOL>Log.debug("<STR_LIT>" % str(mod))<EOL>return getattr(mod, import_name)<EOL>

Imports and load a class from a given pex file path and python class name

    For example, if you want to get a class called `Sample` in
    /some-path/sample.pex/heron/examples/src/python/sample.py,
    ``path_to_pex`` needs to be ``/some-path/sample.pex``, and
    ``python_class_name`` needs to be ``heron.examples.src.python.sample.Sample``


<EOL>for handler in Log.handlers:<EOL><INDENT>if isinstance(handler, logging.StreamHandler):<EOL><INDENT>Log.handlers.remove(handler)<EOL><DEDENT><DEDENT>Log.setLevel(level)<EOL>if logfile is not None:<EOL><INDENT>log_format = "<STR_LIT>"<EOL>formatter = logging.Formatter(fmt=log_format, datefmt=date_format)<EOL>file_handler = logging.FileHandler(logfile)<EOL>file_handler.setFormatter(formatter)<EOL>Log.addHandler(file_handler)<EOL><DEDENT>else:<EOL><INDENT>log_format = "<STR_LIT>"<EOL>formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)<EOL>stream_handler = logging.StreamHandler()<EOL>stream_handler.setFormatter(formatter)<EOL>Log.addHandler(stream_handler)<EOL><DEDENT>

Configure logger which dumps log on terminal

    :param level: logging level: info, warning, verbose...
    :type level: logging level
    :param logfile: log file name, default to None
    :type logfile: string
    :return: None
    :rtype: None


def init_rotating_logger(level, logfile, max_files, max_bytes):

logging.basicConfig()<EOL>root_logger = logging.getLogger()<EOL>log_format = "<STR_LIT>"<EOL>root_logger.setLevel(level)<EOL>handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)<EOL>handler.setFormatter(logging.Formatter(fmt=log_format, datefmt=date_format))<EOL>root_logger.addHandler(handler)<EOL>for handler in root_logger.handlers:<EOL><INDENT>root_logger.debug("<STR_LIT>" + str(handler))<EOL>if isinstance(handler, logging.StreamHandler):<EOL><INDENT>root_logger.debug("<STR_LIT>" + str(handler))<EOL>root_logger.handlers.remove(handler)<EOL><DEDENT><DEDENT>

Initializes a rotating logger

    It also makes sure that any StreamHandler is removed, so as to avoid stdout/stderr
    constipation issues


if '<STR_LIT>' in cl_args and cl_args['<STR_LIT>']:<EOL><INDENT>configure(logging.DEBUG)<EOL><DEDENT>else:<EOL><INDENT>configure(logging.INFO)<EOL><DEDENT>

simply set verbose level based on command-line args

    :param cl_args: CLI arguments
    :type cl_args: dict
    :return: None
    :rtype: None


while <NUM_LIT:1>:<EOL><INDENT>line = fileno.readline()<EOL>handler(line)<EOL>if not line:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Stream handling each line from fileno
    :param filno: file object
    :param handler: a function that will be called for each line from fileno
    :return: None


_stream_process_fileno(process.stdout, handler)<EOL>

Stream the stdout for a process out to display
    :param process: the process to stream the stdout for
    :param handler: a function that will be called for each stdout line
    :return: None


_stream_process_fileno(process.stderr, handler)<EOL>

Stream the stderr for a process out to display
    :param process: the process to stream the stderr for
    :param handler: a function that will be called for each stderr line
    :return: None


def _async_stream_process_output(process, stream_fn, handler):

logging_thread = Thread(target=stream_fn, args=(process, handler, ))<EOL>logging_thread.daemon = True<EOL>logging_thread.start()<EOL>return logging_thread<EOL>

Stream and handle the output of a process
    :param process: the process to stream the output for
    :param stream_fn: the function that applies handler to process
    :param handler: a function that will be called for each log line
    :return: None


return _async_stream_process_output(process, stream_process_stdout, handler)<EOL>

Stream and handler the stdout of a process
    :param process: the process to stream the stdout for
    :param handler: a function that will be called to handle each line
    :return: None


return _async_stream_process_output(process, stream_process_stderr, handler)<EOL>

Stream and handler the stderr of a process
    :param process: the process to stream the stderr for
    :param handler: a function that will be called to handle each line
    :return: None


stdout_builder = StringBuilder()<EOL>async_stream_process_stdout(proc, stdout_builder.add)<EOL>return stdout_builder<EOL>

Save stdout into string builder
    :param proc: the process to save stdout for
    :return StringBuilder


stderr_builder = StringBuilder()<EOL>async_stream_process_stderr(proc, stderr_builder.add)<EOL>return stderr_builder<EOL>

Save stderr into string builder
    :param proc: the process to save stderr for
    :return StringBuilder


return async_stdout_builder(proc), async_stderr_builder(proc)<EOL>

Save stdout and stderr into string builders
    :param proc: the process to save stdout and stderr for
    :return (StringBuilder, StringBuilder)


size = os.path.getsize(filename)<EOL>with open(filename, "<STR_LIT:rb>") as f:<EOL><INDENT>fm = mmap.mmap(f.fileno(), <NUM_LIT:0>, mmap.MAP_SHARED, mmap.PROT_READ)<EOL>try:<EOL><INDENT>for i in range(size - <NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>if fm[i] == '<STR_LIT:\n>':<EOL><INDENT>n -= <NUM_LIT:1><EOL>if n == -<NUM_LIT:1>:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>return fm[i + <NUM_LIT:1> if i else <NUM_LIT:0>:].splitlines()<EOL><DEDENT>finally:<EOL><INDENT>fm.close()<EOL><DEDENT><DEDENT>

Returns last n lines from the filename. No exception handling


def traverse(element, query, deep=False):

<EOL>part = query[<NUM_LIT:0>]<EOL>if not part:<EOL><INDENT>query = query[<NUM_LIT:1>:]<EOL>part = query[<NUM_LIT:0>]<EOL>deep = True<EOL><DEDENT>part, predicate = xpath_re.match(query[<NUM_LIT:0>]).groups()<EOL>for c in element._children:<EOL><INDENT>if part in ('<STR_LIT:*>', c.tagname) and c._match(predicate):<EOL><INDENT>if len(query) == <NUM_LIT:1>:<EOL><INDENT>yield c<EOL><DEDENT>else:<EOL><INDENT>for e in traverse(c, query[<NUM_LIT:1>:]):<EOL><INDENT>yield e<EOL><DEDENT><DEDENT><DEDENT>if deep:<EOL><INDENT>for e in traverse(c, query, deep=True):<EOL><INDENT>yield e<EOL><DEDENT><DEDENT><DEDENT>

Helper function to traverse an element tree rooted at element, yielding nodes matching the query.


parts = query.split('<STR_LIT:/>')<EOL>norm = []<EOL>for p in parts:<EOL><INDENT>p = p.strip()<EOL>if p:<EOL><INDENT>norm.append(p)<EOL><DEDENT>elif '<STR_LIT>' not in norm:<EOL><INDENT>norm.append('<STR_LIT>')<EOL><DEDENT><DEDENT>return norm<EOL>

Given a simplified XPath query string, returns an array of normalized query parts.


def parse(url_or_path, encoding=None, handler_class=DrillHandler):

handler = handler_class()<EOL>parser = expat.ParserCreate(encoding)<EOL>parser.buffer_text = <NUM_LIT:1><EOL>parser.StartElementHandler = handler.start_element<EOL>parser.EndElementHandler = handler.end_element<EOL>parser.CharacterDataHandler = handler.characters<EOL>if isinstance(url_or_path, basestring):<EOL><INDENT>if '<STR_LIT>' in url_or_path[:<NUM_LIT:20>]:<EOL><INDENT>with contextlib.closing(url_lib.urlopen(url_or_path)) as f:<EOL><INDENT>parser.ParseFile(f)<EOL><DEDENT><DEDENT>elif url_or_path[:<NUM_LIT:100>].strip().startswith('<STR_LIT:<>'):<EOL><INDENT>if isinstance(url_or_path, unicode):<EOL><INDENT>if encoding is None:<EOL><INDENT>encoding = '<STR_LIT:utf-8>'<EOL><DEDENT>url_or_path = url_or_path.encode(encoding)<EOL><DEDENT>parser.Parse(url_or_path, True)<EOL><DEDENT>else:<EOL><INDENT>with open(url_or_path, '<STR_LIT:rb>') as f:<EOL><INDENT>parser.ParseFile(f)<EOL><DEDENT><DEDENT><DEDENT>elif PY3 and isinstance(url_or_path, bytes):<EOL><INDENT>parser.ParseFile(bytes_io(url_or_path))<EOL><DEDENT>else:<EOL><INDENT>parser.ParseFile(url_or_path)<EOL><DEDENT>return handler.root<EOL>

:param url_or_path: A file-like object, a filesystem path, a URL, or a string containing XML
:rtype: :class:`XmlElement`


def iterparse(filelike, encoding=None, handler_class=DrillHandler, xpath=None):

parser = expat.ParserCreate(encoding)<EOL>elem_iter = DrillElementIterator(filelike, parser)<EOL>handler = handler_class(elem_iter, xpath)<EOL>parser.buffer_text = <NUM_LIT:1><EOL>parser.StartElementHandler = handler.start_element<EOL>parser.EndElementHandler = handler.end_element<EOL>parser.CharacterDataHandler = handler.characters<EOL>return elem_iter<EOL>

:param filelike: A file-like object with a ``read`` method
:returns: An iterator yielding :class:`XmlElement` objects


for e in self:<EOL><INDENT>return e<EOL><DEDENT>

Returns the first matching element of this query, or None if there was no match.


last_found = None<EOL>for e in self:<EOL><INDENT>last_found = e<EOL><DEDENT>return last_found<EOL>

Returns the last matching element of this query, or None if there was no match.


return True<EOL>

If we exist, we should evaluate to True, even if __len__ returns 0.


return len(self._children)<EOL>


if isinstance(idx, basestring):<EOL><INDENT>return self.attrs[idx]<EOL><DEDENT>return self._children[idx]<EOL>

Returns the child node at the given index.


return self.first(name)<EOL>

Allows access to any attribute or child node directly.


multiline = bool(self._children)<EOL>newline_start = multiline and not bool(self.data)<EOL>writer.start(self.tagname, self.attrs, newline=newline_start)<EOL>if self.data:<EOL><INDENT>writer.data(self.data, newline=bool(self._children))<EOL><DEDENT>for c in self._children:<EOL><INDENT>c.write(writer)<EOL><DEDENT>writer.end(self.tagname, indent=multiline)<EOL>

Writes an XML representation of this node (including descendants) to the specified file-like object.

:param writer: An :class:`XmlWriter` instance to write this node to


s = bytes_io()<EOL>writer = XmlWriter(s, **kwargs)<EOL>self.write(writer)<EOL>return s.getvalue()<EOL>

Returns an XML representation of this node (including descendants). This method automatically creates an
:class:`XmlWriter` instance internally to handle the writing.

:param **kwargs: Any named arguments are passed along to the :class:`XmlWriter` constructor


def append(self, name, attrs=None, data=None):

elem = self.__class__(name, attrs, data, parent=self, index=len(self._children))<EOL>self._children.append(elem)<EOL>return elem<EOL>

Called when the parser detects a start tag (child element) while in this node. Internally creates an
:class:`XmlElement` and adds it to the end of this node's children.

:param name: The tag name to add
:param attrs: Attributes for the new tag
:param data: CDATA for the new tag
:returns: The newly-created element
:rtype: :class:`XmlElement`


def insert(self, before, name, attrs=None, data=None):

if isinstance(before, self.__class__):<EOL><INDENT>if before.parent != self:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>before = before.index<EOL><DEDENT>before = min(max(<NUM_LIT:0>, before), len(self._children))<EOL>elem = self.__class__(name, attrs, data, parent=self, index=before)<EOL>self._children.insert(before, elem)<EOL>for idx, c in enumerate(self._children):<EOL><INDENT>c.index = idx<EOL><DEDENT>return elem<EOL>

Inserts a new element as a child of this element, before the specified index or sibling.

:param before: An :class:`XmlElement` or a numeric index to insert the new node before
:param name: The tag name to add
:param attrs: Attributes for the new tag
:param data: CDATA for the new tag
:returns: The newly-created element
:rtype: :class:`XmlElement`


self.attrs = {}<EOL>self.data = '<STR_LIT>'<EOL>self._children = []<EOL>

Clears out all children, attributes, and data. Especially useful when using :func:`iterparse`, to release
memory used by storing attributes, character data, and child nodes.


for key in sorted(self.attrs):<EOL><INDENT>yield key, self.attrs[key]<EOL><DEDENT>

A generator yielding ``(key, value)`` attribute pairs, sorted by key name.


def children(self, name=None, reverse=False):

elems = self._children<EOL>if reverse:<EOL><INDENT>elems = reversed(elems)<EOL><DEDENT>for elem in elems:<EOL><INDENT>if name is None or elem.tagname == name:<EOL><INDENT>yield elem<EOL><DEDENT><DEDENT>

A generator yielding children of this node.

:param name: If specified, only consider elements with this tag name
:param reverse: If ``True``, children will be yielded in reverse declaration order


if not pred:<EOL><INDENT>return True<EOL><DEDENT>pred = pred[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>if pred.startswith('<STR_LIT:@>'):<EOL><INDENT>pred = pred[<NUM_LIT:1>:]<EOL>if '<STR_LIT:=>' in pred:<EOL><INDENT>attr, value = pred.split('<STR_LIT:=>', <NUM_LIT:1>)<EOL>if value[<NUM_LIT:0>] in ('<STR_LIT:">', "<STR_LIT:'>"):<EOL><INDENT>value = value[<NUM_LIT:1>:]<EOL><DEDENT>if value[-<NUM_LIT:1>] in ('<STR_LIT:">', "<STR_LIT:'>"):<EOL><INDENT>value = value[:-<NUM_LIT:1>]<EOL><DEDENT>return self.attrs.get(attr) == value<EOL><DEDENT>else:<EOL><INDENT>return pred in self.attrs<EOL><DEDENT><DEDENT>elif num_re.match(pred):<EOL><INDENT>index = int(pred)<EOL>if index < <NUM_LIT:0>:<EOL><INDENT>if self.parent:<EOL><INDENT>return self.index == (len(self.parent._children) + index)<EOL><DEDENT>else:<EOL><INDENT>return index == <NUM_LIT:0><EOL><DEDENT><DEDENT>else:<EOL><INDENT>return index == self.index<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if '<STR_LIT:=>' in pred:<EOL><INDENT>tag, value = pred.split('<STR_LIT:=>', <NUM_LIT:1>)<EOL>if value[<NUM_LIT:0>] in ('<STR_LIT:">', "<STR_LIT:'>"):<EOL><INDENT>value = value[<NUM_LIT:1>:]<EOL><DEDENT>if value[-<NUM_LIT:1>] in ('<STR_LIT:">', "<STR_LIT:'>"):<EOL><INDENT>value = value[:-<NUM_LIT:1>]<EOL><DEDENT>for c in self._children:<EOL><INDENT>if c.tagname == tag and c.data == value:<EOL><INDENT>return True<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for c in self._children:<EOL><INDENT>if c.tagname == pred:<EOL><INDENT>return True<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return False<EOL>

Helper function to determine if this node matches the given predicate.


path = '<STR_LIT>' % (self.tagname, self.index or <NUM_LIT:0>)<EOL>p = self.parent<EOL>while p is not None:<EOL><INDENT>if p.parent or include_root:<EOL><INDENT>path = '<STR_LIT>' % (p.tagname, p.index or <NUM_LIT:0>, path)<EOL><DEDENT>p = p.parent<EOL><DEDENT>return path<EOL>

Returns a canonical path to this element, relative to the root node.

:param include_root: If ``True``, include the root node in the path. Defaults to ``False``.


return XmlQuery(self, query)<EOL>

Recursively find any descendants of this node matching the given query.

:param query: A simplified XPath query describing elements that should be returned, e.g. ``//title``,
    ``book/*``, ``*/author``, ``*/*``, etc.
:returns: An :class:`XmlQuery` yielding matching descendants


for c in self._children:<EOL><INDENT>if name is None or c.tagname == name:<EOL><INDENT>yield c<EOL><DEDENT>for gc in c.find(name):<EOL><INDENT>yield gc<EOL><DEDENT><DEDENT>

Recursively find any descendants of this node with the given tag name. If a tag name is omitted, this will
yield every descendant node.

:param name: If specified, only consider elements with this tag name
:returns: A generator yielding descendants of this node


for c in self.children(name):<EOL><INDENT>return c<EOL><DEDENT>

Returns the first child of this node.

:param name: If specified, only consider elements with this tag name
:rtype: :class:`XmlElement`


for c in self.children(name, reverse=True):<EOL><INDENT>return c<EOL><DEDENT>

Returns the last child of this node.

:param name: If specified, only consider elements with this tag name
:rtype: :class:`XmlElement`


p = self.parent<EOL>while p is not None:<EOL><INDENT>if name is None or p.tagname == name:<EOL><INDENT>yield p<EOL><DEDENT>p = p.parent<EOL><DEDENT>

Yields all parents of this element, back to the root element.

:param name: If specified, only consider elements with this tag name


if self.parent and self.index:<EOL><INDENT>for c in self.parent._children:<EOL><INDENT>if c.index != self.index and (name is None or name == c.tagname):<EOL><INDENT>yield c<EOL><DEDENT><DEDENT><DEDENT>

Yields all siblings of this node (not including the node itself).

:param name: If specified, only consider elements with this tag name


if self.parent is None or self.index is None:<EOL><INDENT>return None<EOL><DEDENT>for idx in xrange(self.index + <NUM_LIT:1>, len(self.parent)):<EOL><INDENT>if name is None or self.parent[idx].tagname == name:<EOL><INDENT>return self.parent[idx]<EOL><DEDENT><DEDENT>

Returns the next sibling of this node.

:param name: If specified, only consider elements with this tag name
:rtype: :class:`XmlElement`


if self.parent is None or self.index is None:<EOL><INDENT>return None<EOL><DEDENT>for idx in xrange(self.index - <NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>if name is None or self.parent[idx].tagname == name:<EOL><INDENT>return self.parent[idx]<EOL><DEDENT><DEDENT>

Returns the previous sibling of this node.

:param name: If specified, only consider elements with this tag name
:rtype: :class:`XmlElement`


self.tags = list(set(self.tags or []) | set([tag]))<EOL>

Adds a tag to the list of tags and makes sure the result list contains only unique results.


self.tags = list(set(self.tags or []) - set([tag]))<EOL>


result = super(JackalDoc, self).to_dict(include_meta=include_meta)<EOL>if include_meta:<EOL><INDENT>source = result.pop('<STR_LIT>')<EOL>return {**result, **source}<EOL><DEDENT>else:<EOL><INDENT>return result<EOL><DEDENT>

Returns the result as a dictionary, provide the include_meta flag to als show information like index and doctype.


return input(question + '<STR_LIT>'.format(default)) or default<EOL>

Helper function to return default value if string is empty.


print("<STR_LIT>")<EOL>mapping = { '<STR_LIT:1>': '<STR_LIT:y>', '<STR_LIT:0>': '<STR_LIT:n>'}<EOL>config = Config()<EOL>host = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT:host>'))<EOL>config.set('<STR_LIT>', '<STR_LIT:host>', host)<EOL>if input_with_default("<STR_LIT>", mapping[config.get('<STR_LIT>', '<STR_LIT>')]) == '<STR_LIT:y>':<EOL><INDENT>config.set('<STR_LIT>', '<STR_LIT>', '<STR_LIT:1>')<EOL>if input_with_default("<STR_LIT>", '<STR_LIT:y>') == '<STR_LIT:y>':<EOL><INDENT>ca_certs = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT>', ca_certs)<EOL><DEDENT>else:<EOL><INDENT>config.set('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>config.set('<STR_LIT>', '<STR_LIT>', '<STR_LIT:0>')<EOL><DEDENT>if input_with_default("<STR_LIT>", mapping[config.get('<STR_LIT>', '<STR_LIT>')]) == '<STR_LIT:y>':<EOL><INDENT>config.set('<STR_LIT>', '<STR_LIT>', '<STR_LIT:1>')<EOL>client_cert = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT>', client_cert)<EOL>client_key = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT>', client_key)<EOL><DEDENT>else:<EOL><INDENT>config.set('<STR_LIT>', '<STR_LIT>', '<STR_LIT:0>')<EOL><DEDENT>index = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT:index>'))<EOL>config.set('<STR_LIT>', '<STR_LIT:index>', index)<EOL>initialize_indices = (input_with_default("<STR_LIT>", '<STR_LIT:y>').lower() == '<STR_LIT:y>')<EOL>nmap_dir = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>if not os.path.exists(nmap_dir):<EOL><INDENT>os.makedirs(nmap_dir)<EOL><DEDENT>config.set('<STR_LIT>', '<STR_LIT>', nmap_dir)<EOL>nmap_options = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT>', nmap_options)<EOL>configure_nessus = (input_with_default("<STR_LIT>", '<STR_LIT:n>').lower() == '<STR_LIT:y>')<EOL>if configure_nessus:<EOL><INDENT>nessus_host = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT:host>'))<EOL>nessus_template = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>nessus_access = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>nessus_secret = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT:host>', nessus_host)<EOL>config.set('<STR_LIT>', '<STR_LIT>', nessus_template)<EOL>config.set('<STR_LIT>', '<STR_LIT>', nessus_access)<EOL>config.set('<STR_LIT>', '<STR_LIT>', nessus_secret)<EOL><DEDENT>configure_pipes = (input_with_default("<STR_LIT>", '<STR_LIT:n>').lower() == '<STR_LIT:y>')<EOL>if configure_pipes:<EOL><INDENT>directory = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT>', directory)<EOL>config_file = input_with_default("<STR_LIT>", config.get('<STR_LIT>', '<STR_LIT>'))<EOL>config.set('<STR_LIT>', '<STR_LIT>', config_file)<EOL>if not os.path.exists(directory):<EOL><INDENT>create = (input_with_default("<STR_LIT>", '<STR_LIT:n>').lower() == '<STR_LIT:y>')<EOL>if create:<EOL><INDENT>os.makedirs(directory)<EOL><DEDENT><DEDENT>if not os.path.exists(os.path.join(config.config_dir, config_file)):<EOL><INDENT>f = open(os.path.join(config.config_dir, config_file), '<STR_LIT:a>')<EOL>f.close()<EOL><DEDENT><DEDENT>config.write_config(initialize_indices)<EOL>


if not section in self.config:<EOL><INDENT>self.config.add_section(section)<EOL><DEDENT>self.config.set(section, key, value)<EOL>

Creates the section value if it does not exists and sets the value.
Use write_config to actually set the value.