def __init__(self, name, parameters, directory="<STR_LIT:.>", tag_prefix="<STR_LIT>", description=None):

<EOL>self.__experiment_name = name + "<STR_LIT:@>" + time.strftime("<STR_LIT>") + '<STR_LIT:.>' + str(random.randint(<NUM_LIT:0>, <NUM_LIT:100>))<EOL>self.__results_recorded = False<EOL>self.__repository_directory = directory<EOL>if tag_prefix[-<NUM_LIT:1>] != '<STR_LIT:/>':<EOL><INDENT>tag_prefix += '<STR_LIT:/>'<EOL><DEDENT>self.__tag_name = tag_prefix + self.__experiment_name<EOL>self.__parameters = parameters<EOL>self.__description = description<EOL>

Start logging a new experiment.
:param name: the name of the experiment
:type name: str
:param parameters: a dictionary with all the parameters of the experiment.
:type parameters: dict
:param directory: a string of the directory of the git repository, where the experiment will be logged.
:type directory: str
:param tag_prefix: the prefix of the "folder" where the experiment-related tags will be placed
:type tag_prefix: str


repository = Repo(self.__repository_directory, search_parent_directories=True)<EOL>for tag in repository.tags:<EOL><INDENT>if tag.name == self.__tag_name:<EOL><INDENT>tag_object = tag<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>data = json.loads(tag_object.tag.message)<EOL>data["<STR_LIT>"] = results<EOL>TagReference.create(repository, self.__tag_name, message=json.dumps(data),<EOL>ref=tag_object.tag.object, force=True)<EOL>self.__results_recorded = True<EOL>

Record the results of this experiment, by updating the tag.
:param results: A dictionary containing the results of the experiment.
:type results: dict


def record_results_and_push(self, results, remote_name='<STR_LIT>'):

<EOL>raise NotImplemented()<EOL>

Record the results of this experiment, by updating the tag and push to remote
:param results:  A dictionary containing the results of the experiment.
:type results: dict
:param remote_name: the name of the remote to push the tags (or None for the default)
:type remote_name: str


assert self.__tag_name not in [t.name for t in repository.tags]<EOL>return TagReference.create(repository, self.__tag_name, message=json.dumps(data))<EOL>

Tag the current repository.
:param data: a dictionary containing the data about the experiment
:type data: dict


for root, dirs, files in os.walk(repository.working_dir):<EOL><INDENT>for f in files:<EOL><INDENT>relative_path = os.path.join(root, f)[len(repository.working_dir) + <NUM_LIT:1>:]<EOL>try:<EOL><INDENT>repository.head.commit.tree[relative_path] <EOL>yield relative_path<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>

:return: the files that have been modified and can be added


repository = Repo(self.__repository_directory, search_parent_directories=True)<EOL>if len(repository.untracked_files) > <NUM_LIT:0>:<EOL><INDENT>logging.warning("<STR_LIT>", repository.untracked_files)<EOL><DEDENT>current_commit = repository.head.commit<EOL>started_state_is_dirty = repository.is_dirty()<EOL>if started_state_is_dirty:<EOL><INDENT>repository.index.add([p for p in self.__get_files_to_be_added(repository)])<EOL>commit_obj = repository.index.commit("<STR_LIT>" + self.__experiment_name)<EOL>sha = commit_obj.hexsha<EOL><DEDENT>else:<EOL><INDENT>sha = repository.head.object.hexsha<EOL><DEDENT>data = {"<STR_LIT>": parameters, "<STR_LIT>": time.time(), "<STR_LIT:description>": self.__description,<EOL>"<STR_LIT>": sha}<EOL>tag_object = self.__tag_repo(data, repository)<EOL>if started_state_is_dirty:<EOL><INDENT>repository.head.reset(current_commit, working_tree=False, index=True)<EOL><DEDENT>return tag_object<EOL>

Start an experiment by capturing the state of the code
:param parameters: a dictionary containing the parameters of the experiment
:type parameters: dict
:return: the tag representing this experiment
:rtype: TagReference


while os.path.islink(src):<EOL><INDENT>src = os.path.join(os.path.dirname(src), os.readlink(src))<EOL><DEDENT>return src<EOL>

Resolve all symbolic references that `src` points to.  Note that this
is different than `os.path.realpath` as path components leading up to
the final location may still be symbolic links.


return json.loads(manifest_text)<EOL>

Parse a json manifest.

    Manifest format:
      {
        "modules": [ {"src": "path_on_disk", "dest": "path_in_pex"}, ... ],
        "resources": [ {"src": "path_on_disk", "dest": "path_in_pex"}, ... ],
        "prebuiltLibraries": [ "path1", "path2", ... ],
        "requirements": [ "thing1", "thing2==version", ... ],
      }


myindex = context.get_partition_index()<EOL>self._files_to_consume = self._files[myindex::context.get_num_partitions()]<EOL>self.logger.info("<STR_LIT>" % self._files_to_consume)<EOL>self._lines_to_consume = self._get_next_lines()<EOL>self._emit_count = <NUM_LIT:0><EOL>

Implements TextFile Generator's setup method


self.logger.info("<STR_LIT>")<EOL>self.logger.info("<STR_LIT>" % str(config))<EOL>self.logger.info("<STR_LIT>" % str(context))<EOL>self.emit_count = <NUM_LIT:0><EOL>self.ack_count = <NUM_LIT:0><EOL>self.fail_count = <NUM_LIT:0><EOL>if not PulsarSpout.serviceUrl in config or not PulsarSpout.topicName in config:<EOL><INDENT>self.logger.fatal("<STR_LIT>")<EOL><DEDENT>self.pulsar_cluster = str(config[PulsarSpout.serviceUrl])<EOL>self.topic = str(config[PulsarSpout.topicName])<EOL>mode = config[api_constants.TOPOLOGY_RELIABILITY_MODE]<EOL>if mode == api_constants.TopologyReliabilityMode.ATLEAST_ONCE:<EOL><INDENT>self.acking_timeout = <NUM_LIT:1000> * int(config[api_constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS])<EOL><DEDENT>else:<EOL><INDENT>self.acking_timeout = <NUM_LIT><EOL><DEDENT>if PulsarSpout.receiveTimeoutMs in config:<EOL><INDENT>self.receive_timeout_ms = config[PulsarSpout.receiveTimeoutMs]<EOL><DEDENT>else:<EOL><INDENT>self.receive_timeout_ms = <NUM_LIT:10><EOL><DEDENT>if PulsarSpout.deserializer in config:<EOL><INDENT>self.deserializer = config[PulsarSpout.deserializer]<EOL>if not callable(self.deserializer):<EOL><INDENT>self.logger.fatal("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.deserializer = self.default_deserializer<EOL><DEDENT>self.logConfFileName = GenerateLogConfig(context)<EOL>self.logger.info("<STR_LIT>" % self.logConfFileName)<EOL>self.client = pulsar.Client(self.pulsar_cluster, log_conf_file_path=self.logConfFileName)<EOL>self.logger.info("<STR_LIT>" % self.pulsar_cluster)<EOL>try:<EOL><INDENT>self.consumer = self.client.subscribe(self.topic, context.get_topology_name(),<EOL>consumer_type=pulsar.ConsumerType.Failover,<EOL>unacked_messages_timeout_ms=self.acking_timeout)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.logger.fatal("<STR_LIT>" % str(e))<EOL><DEDENT>self.logger.info("<STR_LIT>" % self.topic)<EOL>

Implements Pulsar Spout's initialize method


raise NotImplementedError("<STR_LIT>")<EOL>

Called when a task for this operator is initialized within a worker on the cluster

        It provides the operator with the environment in which the bolt executes.
        Note that ``__init__()`` should not be overriden for initialization of a bolt, as it is used
        internally by BaseBolt; instead, ``initialize()`` should be used to initialize any custom
        variables or connection to databases.

        :type context: :class:`Context`
        :param context: This object can be used to get information about this task's place within the
                        topology, including the task id and component id of this task, input and output
                        information, etc.
        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


Process a single tuple of input

        The Tuple object contains metadata on it about which component/stream/task it came from.
        To emit a tuple, call ``context.emit(tuple)``.

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**

        :type tup: :class:`Tuple`
        :param tup: Tuple to process


source_streamlet = None<EOL>if callable(source):<EOL><INDENT>source_streamlet = SupplierStreamlet(source)<EOL><DEDENT>elif isinstance(source, Generator):<EOL><INDENT>source_streamlet = GeneratorStreamlet(source)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>self._sources.append(source_streamlet)<EOL>return source_streamlet<EOL>

Adds a new source to the computation DAG


stage_names = sets.Set()<EOL>for source in self._sources:<EOL><INDENT>source._build(bldr, stage_names)<EOL><DEDENT>for source in self._sources:<EOL><INDENT>if not source._all_built():<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>

Builds the topology and returns the builder


pass<EOL>

Fetches the task id of the current instance of the operator


pass<EOL>


pass<EOL>

Fetches the stream name that we are operating on


pass<EOL>

Fetches the number of partitions of the stream we are operating on


pass<EOL>

Fetches the partition of the stream that we are operating on


pass<EOL>

The state where components can store any of their local state


self._name = name<EOL>return self<EOL>


self._num_partitions = num_partitions<EOL>return self<EOL>


from heronpy.streamlet.impl.mapbolt import MapStreamlet<EOL>map_streamlet = MapStreamlet(map_function, self)<EOL>self._add_child(map_streamlet)<EOL>return map_streamlet<EOL>

Return a new Streamlet by applying map_function to each element of this Streamlet.


from heronpy.streamlet.impl.flatmapbolt import FlatMapStreamlet<EOL>fm_streamlet = FlatMapStreamlet(flatmap_function, self)<EOL>self._add_child(fm_streamlet)<EOL>return fm_streamlet<EOL>

Return a new Streamlet by applying map_function to each element of this Streamlet
           and flattening the result


from heronpy.streamlet.impl.filterbolt import FilterStreamlet<EOL>filter_streamlet = FilterStreamlet(filter_function, self)<EOL>self._add_child(filter_streamlet)<EOL>return filter_streamlet<EOL>

Return a new Streamlet containing only the elements that satisfy filter_function


def repartition(self, num_partitions, repartition_function=None):

from heronpy.streamlet.impl.repartitionbolt import RepartitionStreamlet<EOL>if repartition_function is None:<EOL><INDENT>repartition_function = lambda x: x<EOL><DEDENT>repartition_streamlet = RepartitionStreamlet(num_partitions, repartition_function, self)<EOL>self._add_child(repartition_streamlet)<EOL>return repartition_streamlet<EOL>

Return a new Streamlet containing all elements of the this streamlet but having
        num_partitions partitions. Note that this is different from num_partitions(n) in
        that new streamlet will be created by the repartition call.
        If repartiton_function is not None, it is used to decide which parititons
        (from 0 to num_partitions -1), it should route each element to.
        It could also return a list of partitions if it wants to send it to multiple
        partitions.


retval = []<EOL>for i in range(num_clones):<EOL><INDENT>retval.append(self.repartition(self.get_num_partitions()))<EOL><DEDENT>return retval<EOL>

Return num_clones number of streamlets each containing all elements
        of the current streamlet


def reduce_by_window(self, window_config, reduce_function):

from heronpy.streamlet.impl.reducebywindowbolt import ReduceByWindowStreamlet<EOL>reduce_streamlet = ReduceByWindowStreamlet(window_config, reduce_function, self)<EOL>self._add_child(reduce_streamlet)<EOL>return reduce_streamlet<EOL>

Return a new Streamlet in which each element of this Streamlet are collected
          over a window defined by window_config and then reduced using the reduce_function
          reduce_function takes two element at one time and reduces them to one element that
          is used in the subsequent operations.


from heronpy.streamlet.impl.unionbolt import UnionStreamlet<EOL>union_streamlet = UnionStreamlet(self, other_streamlet)<EOL>self._add_child(union_streamlet)<EOL>other_streamlet._add_child(union_streamlet)<EOL>return union_streamlet<EOL>

Returns a new Streamlet that consists of elements of both this and other_streamlet


from heronpy.streamlet.impl.transformbolt import TransformStreamlet<EOL>transform_streamlet = TransformStreamlet(transform_operator, self)<EOL>self._add_child(transform_streamlet)<EOL>return transform_streamlet<EOL>

Returns a  new Streamlet by applying the transform_operator on each element of this
        streamlet. The transform_function is of the type TransformOperator.
        Before starting to cycle over the Streamlet, the open function of the transform_operator is
        called. This allows the transform_operator to do any kind of initialization/loading, etc.


from heronpy.streamlet.impl.logbolt import LogStreamlet<EOL>log_streamlet = LogStreamlet(self)<EOL>self._add_child(log_streamlet)<EOL>return<EOL>

Logs all elements of this streamlet. This returns nothing


from heronpy.streamlet.impl.consumebolt import ConsumeStreamlet<EOL>consume_streamlet = ConsumeStreamlet(consume_function, self)<EOL>self._add_child(consume_streamlet)<EOL>return<EOL>

Calls consume_function for each element of this streamlet. This function returns nothing


def join(self, join_streamlet, window_config, join_function):

from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt<EOL>join_streamlet_result = JoinStreamlet(JoinBolt.INNER, window_config,<EOL>join_function, self, join_streamlet)<EOL>self._add_child(join_streamlet_result)<EOL>join_streamlet._add_child(join_streamlet_result)<EOL>return join_streamlet_result<EOL>

Return a new Streamlet by joining join_streamlet with this streamlet


def outer_right_join(self, join_streamlet, window_config, join_function):

from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt<EOL>join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_RIGHT, window_config,<EOL>join_function, self, join_streamlet)<EOL>self._add_child(join_streamlet_result)<EOL>join_streamlet._add_child(join_streamlet_result)<EOL>return join_streamlet_result<EOL>

Return a new Streamlet by outer right join_streamlet with this streamlet


def outer_left_join(self, join_streamlet, window_config, join_function):

from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt<EOL>join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_LEFT, window_config,<EOL>join_function, self, join_streamlet)<EOL>self._add_child(join_streamlet_result)<EOL>join_streamlet._add_child(join_streamlet_result)<EOL>return join_streamlet_result<EOL>

Return a new Streamlet by left join_streamlet with this streamlet


def outer_join(self, join_streamlet, window_config, join_function):

from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt<EOL>join_streamlet_result = JoinStreamlet(JoinBolt.OUTER, window_config,<EOL>join_function, self, join_streamlet)<EOL>self._add_child(join_streamlet_result)<EOL>join_streamlet._add_child(join_streamlet_result)<EOL>return join_streamlet_result<EOL>

Return a new Streamlet by outer join_streamlet with this streamlet


def reduce_by_key_and_window(self, window_config, reduce_function):

from heronpy.streamlet.impl.reducebykeyandwindowbolt import ReduceByKeyAndWindowStreamlet<EOL>reduce_streamlet = ReduceByKeyAndWindowStreamlet(window_config, reduce_function, self)<EOL>self._add_child(reduce_streamlet)<EOL>return reduce_streamlet<EOL>

Return a new Streamlet in which each (key, value) pair of this Streamlet are collected
           over the time_window and then reduced using the reduce_function


@abstractmethod<EOL><INDENT>def _build_this(self, builder, stage_names):<DEDENT>

raise RuntimeError("<STR_LIT>")<EOL>

This is the method that's implemented by the operators.
        :type builder: TopologyBuilder
        :param builder: The operator adds in the current streamlet as a spout/bolt


def _default_stage_name_calculator(self, prefix, existing_stage_names):

index = <NUM_LIT:1><EOL>calculated_name = "<STR_LIT>"<EOL>while True:<EOL><INDENT>calculated_name = prefix + "<STR_LIT:->" + str(index)<EOL>if calculated_name not in existing_stage_names:<EOL><INDENT>return calculated_name<EOL><DEDENT>index = index + <NUM_LIT:1><EOL><DEDENT>return "<STR_LIT>"<EOL>

This is the method that's implemented by the operators to get the name of the Streamlet
        :return: The name of the operator


raise NotImplementedError("<STR_LIT>")<EOL>

Called when a task for this operator is initialized within a worker on the cluster

        It provides the operator with the environment in which the operator executes.
        This should be used to initialize any custom variables or connection to databases.

        :type context: :class:`Context`
        :param context: This object can be used to get information about this task's place within the
                        topology, including the task id and component id of this task, input and output
                        information, etc.
        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


Generate the next element
        If there is nothing at the moment to generate, return None

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


pass<EOL>


if not isinstance(name, str):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if not isinstance(config, Config):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if not isinstance(builder, Builder):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>bldr = TopologyBuilder(name=name)<EOL>builder.build(bldr)<EOL>bldr.set_config(config._api_config)<EOL>bldr.build_and_submit()<EOL>


@abstractmethod<EOL><INDENT>def prepare(self, context, component, stream, target_tasks):<DEDENT>

pass<EOL>

Tells the stream grouping at runtime the tasks in the target bolt

        This information should be used in ``choose_tasks()`` to determine the target tasks.

        :param context: topology-worker context
        :param component: source component name
        :param stream: stream id used for this grouping
        :type target_tasks: list of int
        :param target_tasks: list of target task ids


pass<EOL>

Implements a custom stream grouping

        :param values: the values to group on
        :rtype: list of int
        :return: list of task ids to which these values are emitted


specs = {}<EOL>for name, spec in class_dict.items():<EOL><INDENT>if isinstance(spec, HeronComponentSpec):<EOL><INDENT>if spec.name is None:<EOL><INDENT>spec.name = name<EOL><DEDENT>if spec.name in specs:<EOL><INDENT>raise ValueError("<STR_LIT>" % spec.name)<EOL><DEDENT>else:<EOL><INDENT>specs[spec.name] = spec<EOL><DEDENT><DEDENT><DEDENT>return specs<EOL>

Takes a class `__dict__` and returns `HeronComponentSpec` entries


topo_config = {}<EOL>topo_config.update(mcs.DEFAULT_TOPOLOGY_CONFIG)<EOL>for name, custom_config in class_dict.items():<EOL><INDENT>if name == '<STR_LIT>' and isinstance(custom_config, dict):<EOL><INDENT>sanitized_dict = mcs._sanitize_config(custom_config)<EOL>topo_config.update(sanitized_dict)<EOL><DEDENT><DEDENT>return topo_config<EOL>

Takes a class `__dict__` and returns a map containing topology-wide
configuration.

The returned dictionary is a sanitized `dict` of type `<str ->
(str|object)>`.

This classmethod firsts insert default topology configuration and then
overrides it with a given topology-wide configuration. Note that this
configuration will be overriden by a component-specific configuration at
runtime.


@classmethod<EOL><INDENT>def init_topology(mcs, classname, class_dict):<DEDENT>

if classname == '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>heron_options = TopologyType.get_heron_options_from_env()<EOL>initial_state = heron_options.get("<STR_LIT>", "<STR_LIT>")<EOL>tmp_directory = heron_options.get("<STR_LIT>")<EOL>if tmp_directory is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>topology_name = heron_options.get("<STR_LIT>", classname)<EOL>topology_id = topology_name + str(uuid.uuid4())<EOL>topology = topology_pb2.Topology()<EOL>topology.id = topology_id<EOL>topology.name = topology_name<EOL>topology.state = topology_pb2.TopologyState.Value(initial_state)<EOL>topology.topology_config.CopyFrom(TopologyType.get_topology_config_protobuf(class_dict))<EOL>TopologyType.add_bolts_and_spouts(topology, class_dict)<EOL>class_dict['<STR_LIT>'] = topology_name<EOL>class_dict['<STR_LIT>'] = topology_id<EOL>class_dict['<STR_LIT>'] = topology<EOL>class_dict['<STR_LIT>'] = tmp_directory<EOL>class_dict['<STR_LIT>'] = heron_options<EOL>


heron_options_raw = os.environ.get("<STR_LIT>")<EOL>if heron_options_raw is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>options = {}<EOL>for option_line in heron_options_raw.replace("<STR_LIT>", "<STR_LIT:U+0020>").split('<STR_LIT:U+002C>'):<EOL><INDENT>key, sep, value = option_line.partition("<STR_LIT:=>")<EOL>if sep:<EOL><INDENT>options[key] = value<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % option_line)<EOL><DEDENT><DEDENT>return options<EOL>

Retrieves heron options from the `HERON_OPTIONS` environment variable.

        Heron options have the following format:

            cmdline.topologydefn.tmpdirectory=/var/folders/tmpdir
            cmdline.topology.initial.state=PAUSED

        In this case, the returned map will contain:

            #!json
            {
              "cmdline.topologydefn.tmpdirectory": "/var/folders/tmpdir",
              "cmdline.topology.initial.state": "PAUSED"
            }

        Currently supports the following options natively:

        - `cmdline.topologydefn.tmpdirectory`: (required) the directory to which this
        topology's defn file is written
        - `cmdline.topology.initial.state`: (default: "RUNNING") the initial state of the topology
        - `cmdline.topology.name`: (default: class name) topology name on deployment

        Returns: map mapping from key to value


sanitized = {}<EOL>for key, value in custom_config.items():<EOL><INDENT>if not isinstance(key, str):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% (str(type(key)), str(key)))<EOL><DEDENT>if isinstance(value, bool):<EOL><INDENT>sanitized[key] = "<STR_LIT:true>" if value else "<STR_LIT:false>"<EOL><DEDENT>elif isinstance(value, (str, int, float)):<EOL><INDENT>sanitized[key] = str(value)<EOL><DEDENT>else:<EOL><INDENT>sanitized[key] = value<EOL><DEDENT><DEDENT>return sanitized<EOL>

Checks whether a given custom_config and returns a sanitized dict <str -> (str|object)>

        It checks if keys are all strings and sanitizes values of a given dictionary as follows:

        - If string, number or boolean is given as a value, it is converted to string.
          For string and number (int, float), it is converted to string by a built-in ``str()`` method.
          For a boolean value, ``True`` is converted to "true" instead of "True", and ``False`` is
          converted to "false" instead of "False", in order to keep the consistency with
          Java configuration.

        - If neither of the above is given as a value, it is inserted into the sanitized dict as it is.
          These values will need to be serialized before adding to a protobuf message.


if cls.__name__ == '<STR_LIT>':<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>filename = "<STR_LIT>" % cls.topology_name<EOL>path = os.path.join(cls.topologydefn_tmpdir, filename)<EOL>with open(path, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(cls.protobuf_topology.SerializeToString())<EOL><DEDENT>

Writes the Topology .defn file to ``dest``

        This classmethod is meant be used by heron-cli when submitting a topology.


assert name is not None and isinstance(name, str) and name != "<STR_LIT>"<EOL>self.topology_name = name<EOL>self._specs = {}<EOL>self._topology_config = {}<EOL>

Initialize this TopologyBuilder

        :type name: str
        :param name: topology name


for spec in specs:<EOL><INDENT>if not isinstance(spec, HeronComponentSpec):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(spec))<EOL><DEDENT>if spec.name is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if spec.name == "<STR_LIT>":<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if spec.name in self._specs:<EOL><INDENT>raise ValueError("<STR_LIT>" % (spec.name, spec))<EOL><DEDENT>self._specs[spec.name] = spec<EOL><DEDENT>

Add specs to the topology

        :type specs: HeronComponentSpec
        :param specs: specs to add to the topology


def add_spout(self, name, spout_cls, par, config=None, optional_outputs=None):

spout_spec = spout_cls.spec(name=name, par=par, config=config,<EOL>optional_outputs=optional_outputs)<EOL>self.add_spec(spout_spec)<EOL>return spout_spec<EOL>


def add_bolt(self, name, bolt_cls, par, inputs, config=None, optional_outputs=None):

bolt_spec = bolt_cls.spec(name=name, par=par, inputs=inputs, config=config,<EOL>optional_outputs=optional_outputs)<EOL>self.add_spec(bolt_spec)<EOL>return bolt_spec<EOL>


if not isinstance(config, dict):<EOL><INDENT>raise TypeError("<STR_LIT>" % str(config))<EOL><DEDENT>self._topology_config = config<EOL>

Set topology-wide configuration to the topology

        :type config: dict
        :param config: topology-wide config


class_dict = self._construct_topo_class_dict()<EOL>topo_cls = TopologyType(self.topology_name, (Topology,), class_dict)<EOL>topo_cls.write()<EOL>

Builds the topology and submits to the destination


self.value = <NUM_LIT:0><EOL>


self.value += to_add<EOL>


self.value = {}<EOL>


self.incr(key, to_add=<NUM_LIT:0>)<EOL>


def incr(self, key, to_add=<NUM_LIT:1>):

if key not in self.value:<EOL><INDENT>self.value[key] = CountMetric()<EOL><DEDENT>self.value[key].incr(to_add)<EOL>

Increments the value of a given key by ``to_add``


pass<EOL>

Called when this reducer is initialized/reinitialized


pass<EOL>


self.reducer = reducer_cls()<EOL>self.reducer.init()<EOL>

Initializes ReducedMetric object

        :param reducer_cls: IReducer class to use


self.reducer.reduce(value)<EOL>


self.value = {}<EOL>self.reducer = reducer<EOL>

Initializes MultiReducedMetric object

        :param reducer: IReducer class to use


if key not in self.value:<EOL><INDENT>self.value[key] = ReducedMetric(self.reducer)<EOL><DEDENT>self.value[key].update(value)<EOL>

Updates a value of a given key and apply reduction


if key not in self.value:<EOL><INDENT>self.value[key] = ReducedMetric(self.reducer)<EOL><DEDENT>


pass<EOL>

Initializes the state of the function or operator to that of a previous checkpoint.
        This method is invoked when a component is executed as part of a recovery run. In case
        there was no prior state associated with the component, the state will be empty.
        Stateful Spouts/Bolts are expected to hold on to the state variable to save their
        internal state
        Note that init_state() is called before initialize()
        :param state: the previously saved state of the component.


pass<EOL>

This is a hook for the component to perform some actions just before the
        framework saves its state.


@abstractmethod<EOL><INDENT>def put(self, key, value):<DEDENT>

pass<EOL>

Puts {key, value} pair into the state
        :param key: The key to get back the value
        :param value: The value associated with the key


pass<EOL>

Gets the value corresponding to a key
        :param key: The key whose value we want back
        :return: The value associated with the key


pass<EOL>

Allows one to enumerate over the state.
        :return: The enumerate object


pass<EOL>

Gets the task id of this component
        :return: the task_id of this component


pass<EOL>

Gets the component id of this component
        :return: the component_id of this component


pass<EOL>

Returns the cluster config for this component
        Note that the returned config is auto-typed map: <str -> any Python object>.
        :return: the dict of key -> value


pass<EOL>

Returns the name of the topology
        :return: the name of the topology


@abstractmethod<EOL><INDENT>def register_metric(self, name, metric, time_bucket_in_sec):<DEDENT>

pass<EOL>

Registers a new metric to this context
        :param name: The name of the metric
        :param metric: The IMetric that needs to be registered
        :param time_bucket_in_sec: The interval in seconds to do getValueAndReset


pass<EOL>

Returns the declared inputs to specified component
        :param component_id: The name of the component whose inputs we want

        :return: map <streamId namedtuple (same structure as protobuf msg) -> gtype>, or
                 None if not found


return self.get_sources(self.get_component_id())<EOL>

Returns the declared inputs to this component
        :return: map <streamId namedtuple (same structure as protobuf msg) -> gtype>, or
                 None if not found


pass<EOL>

Returns the task ids allocated for the given component id
        :param component_id: The name of the component whose task ids we want

        :return: list of task_ids or None if not found


pass<EOL>

Registers a specified task hook to this context

        :type task_hook: ITaskHook
        :param task_hook: Implementation of ITaskHook


@abstractmethod<EOL><INDENT>def initialize(self, config, context):<DEDENT>

pass<EOL>

Called when a task for this component is initialized within a worker on the cluster

        It provides the bolt with the environment in which the bolt executes.
        Note that ``__init__()`` should not be overriden for initialization of a bolt, as it is used
        internally by BaseBolt; instead, ``initialize()`` should be used to initialize any custom
        variables or connection to databases.

        :type config: dict
        :param config: The Heron configuration for this bolt. This is the configuration provided to
                       the topology merged in with cluster configuration on this machine.
                       Note that types of string values in the config have been automatically converted,
                       meaning that number strings and boolean strings are converted to the appropriate
                       types.
        :type context: :class:`TopologyContext`
        :param context: This object can be used to get information about this task's place within the
                        topology, including the task id and component id of this task, input and output
                        information, etc.


raise NotImplementedError("<STR_LIT>")<EOL>

Process a single tuple of input

        The Tuple object contains metadata on it about which component/stream/task it came from.
        To emit a tuple, call ``self.emit(tuple)``.
        Note that tick tuples are not passed to this method, as the ``process_tick()`` method is
        responsible for processing them.

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**

        :type tup: :class:`Tuple`
        :param tup: Tuple to process


pass<EOL>

Process special tick tuple

        It is compatible with StreamParse API.

        Tick tuples allow time-based behavior to be included in bolts. They will be sent to the bolts
        for which `topology.tick.tuple.freq.secs``
        (or, ``.api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS`` key) is set to an integer value, the
        number of seconds.

        Default behavior is to ignore tick tuples. This method should be overridden by subclasses
        if you want to react to timer events via tick tuples.

        :type tup: :class:`Tuple`
        :param tup: the tick tuple to be processed


@abstractmethod<EOL><INDENT>def processWindow(self, window_info, tuples):<DEDENT>

pass<EOL>

The main interface that needs to be implemented.

        This function is called every WINDOW_SLIDEINTERVAL_SECS seconds
        and contains the data in the last WINDOW_DURATION_SECS seconds
        in a list tuples

        :type window_info: :class:`WindowContext`
        :param window_info: The information about the window

        :type tuples: :class:`list of Tuples`
        :param tuples: The list of tuples in this window


if SlidingWindowBolt.WINDOW_DURATION_SECS in config:<EOL><INDENT>self.window_duration = int(config[SlidingWindowBolt.WINDOW_DURATION_SECS])<EOL><DEDENT>else:<EOL><INDENT>self.logger.fatal("<STR_LIT>")<EOL><DEDENT>if SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS in config:<EOL><INDENT>self.slide_interval = int(config[SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS])<EOL><DEDENT>else:<EOL><INDENT>self.slide_interval = self.window_duration<EOL><DEDENT>if self.slide_interval > self.window_duration:<EOL><INDENT>self.logger.fatal("<STR_LIT>")<EOL><DEDENT>config[api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS] = str(self.slide_interval)<EOL>self.current_tuples = deque()<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>if '<STR_LIT>' in self.saved_state:<EOL><INDENT>self.current_tuples = self.saved_state['<STR_LIT>']<EOL><DEDENT><DEDENT>

We initialize the window duration and slide interval


curtime = int(time.time())<EOL>self.current_tuples.append((tup, curtime))<EOL>self._expire(curtime)<EOL>

Process a single tuple of input

        We add the (time, tuple) pair into our current_tuples. And then look for expiring
        elemnents


curtime = int(time.time())<EOL>window_info = WindowContext(curtime - self.window_duration, curtime)<EOL>tuple_batch = []<EOL>for (tup, tm) in self.current_tuples:<EOL><INDENT>tuple_batch.append(tup)<EOL><DEDENT>self.processWindow(window_info, tuple_batch)<EOL>self._expire(curtime)<EOL>


pass<EOL>

The main interface that needs to be implemented.

        This function is called every WINDOW_DURATION_SECS seconds
        and contains the data in the last WINDOW_DURATION_SECS seconds
        in a list tuples

        :type window_info: :class:`WindowContext`
        :param window_info: The information about the window

        :type tuples: :class:`list of Tuples`
        :param tuples: The list of tuples in this window


if TumblingWindowBolt.WINDOW_DURATION_SECS in config:<EOL><INDENT>self.window_duration = int(config[TumblingWindowBolt.WINDOW_DURATION_SECS])<EOL><DEDENT>else:<EOL><INDENT>self.logger.fatal("<STR_LIT>")<EOL><DEDENT>config[api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS] = str(self.window_duration)<EOL>self.current_tuples = deque()<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>if '<STR_LIT>' in self.saved_state:<EOL><INDENT>self.current_tuples = self.saved_state['<STR_LIT>']<EOL><DEDENT><DEDENT>


Process a single tuple of input

        We simply add the tuple into our current_tuples.


curtime = int(time.time())<EOL>window_info = WindowContext(curtime - self.window_duration, curtime)<EOL>self.processWindow(window_info, list(self.current_tuples))<EOL>for tup in self.current_tuples:<EOL><INDENT>self.ack(tup)<EOL><DEDENT>self.current_tuples.clear()<EOL>


@classmethod<EOL><INDENT>def spec(cls, name=None, inputs=None, par=<NUM_LIT:1>, config=None, optional_outputs=None):<DEDENT>

python_class_path = "<STR_LIT>" % (cls.__module__, cls.__name__)<EOL>if hasattr(cls, '<STR_LIT>'):<EOL><INDENT>_outputs = copy.copy(cls.outputs)<EOL><DEDENT>else:<EOL><INDENT>_outputs = []<EOL><DEDENT>if optional_outputs is not None:<EOL><INDENT>assert isinstance(optional_outputs, (list, tuple))<EOL>for out in optional_outputs:<EOL><INDENT>assert isinstance(out, (str, Stream))<EOL>_outputs.append(out)<EOL><DEDENT><DEDENT>return HeronComponentSpec(name, python_class_path, is_spout=False, par=par,<EOL>inputs=inputs, outputs=_outputs, config=config)<EOL>

Register this bolt to the topology and create ``HeronComponentSpec``

        This method takes an optional ``outputs`` argument for supporting dynamic output fields
        declaration. However, it is recommended that ``outputs`` should be declared as
        an attribute of your ``Bolt`` subclass. Also, some ways of declaring inputs is not supported
        in this implementation; please read the documentation below.

        :type name: str
        :param name: Name of this bolt.
        :type inputs: dict or list
        :param inputs: Streams that feed into this Bolt.

                       Two forms of this are acceptable:

                       1. A `dict` mapping from ``HeronComponentSpec`` to ``Grouping``.
                          In this case, default stream is used.
                       2. A `dict` mapping from ``GlobalStreamId`` to ``Grouping``.
                          This ``GlobalStreamId`` object itself is different from StreamParse, because
                          Heron does not use thrift, although its constructor method is compatible.
                       3. A `list` of ``HeronComponentSpec``. In this case, default stream with
                          SHUFFLE grouping is used.
                       4. A `list` of ``GlobalStreamId``. In this case, SHUFFLE grouping is used.
        :type par: int
        :param par: Parallelism hint for this spout.
        :type config: dict
        :param config: Component-specific config settings.
        :type optional_outputs: list of (str or Stream) or tuple of (str or Stream)
        :param optional_outputs: Additional output fields for this bolt. These fields are added to
                                 existing ``outputs`` class attributes of your bolt. This is an optional
                                 argument, and exists only for supporting dynamic output field
                                 declaration.


def emit(self, tup, stream=Stream.DEFAULT_STREAM_ID,<EOL>anchors=None, direct_task=None, need_task_ids=False):

self.delegate.emit(tup, stream, anchors, direct_task, need_task_ids)<EOL>

Emits a new tuple from this Bolt

        It is compatible with StreamParse API.

        :type tup: list or tuple
        :param tup: the new output Tuple to send from this bolt,
                    which should contain only serializable data.
        :type stream: str
        :param stream: the ID of the stream to emit this Tuple to.
                       Leave empty to emit to the default stream.
        :type anchors: list
        :param anchors: a list of HeronTuples to which the emitted Tuples should be anchored.
        :type direct_task: int
        :param direct_task: the task to send the Tuple to if performing a direct emit.
        :type need_task_ids: bool
        :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.


return tup.stream == TupleHelper.TICK_TUPLE_ID<EOL>

Returns whether or not a given HeronTuple is a tick Tuple

        It is compatible with StreamParse API.


self.delegate.ack(tup)<EOL>

Indicate that processing of a Tuple has succeeded

        It is compatible with StreamParse API.


self.delegate.fail(tup)<EOL>

Indicate that processing of a Tuple has failed

        It is compatible with StreamParse API.


if self.is_spout:<EOL><INDENT>return self._get_spout()<EOL><DEDENT>else:<EOL><INDENT>return self._get_bolt()<EOL><DEDENT>

Returns protobuf message (Spout or Bolt) of this component


spout = topology_pb2.Spout()<EOL>spout.comp.CopyFrom(self._get_base_component())<EOL>self._add_out_streams(spout)<EOL>return spout<EOL>


bolt = topology_pb2.Bolt()<EOL>bolt.comp.CopyFrom(self._get_base_component())<EOL>self._add_in_streams(bolt)<EOL>self._add_out_streams(bolt)<EOL>return bolt<EOL>


comp = topology_pb2.Component()<EOL>comp.name = self.name<EOL>comp.spec = topology_pb2.ComponentObjectSpec.Value("<STR_LIT>")<EOL>comp.class_name = self.python_class_path<EOL>comp.config.CopyFrom(self._get_comp_config())<EOL>return comp<EOL>


proto_config = topology_pb2.Config()<EOL>key = proto_config.kvs.add()<EOL>key.key = TOPOLOGY_COMPONENT_PARALLELISM<EOL>key.value = str(self.parallelism)<EOL>key.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL>if self.custom_config is not None:<EOL><INDENT>sanitized = self._sanitize_config(self.custom_config)<EOL>for key, value in sanitized.items():<EOL><INDENT>if isinstance(value, str):<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.value = value<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>kvs = proto_config.kvs.add()<EOL>kvs.key = key<EOL>kvs.serialized_value = default_serializer.serialize(value)<EOL>kvs.type = topology_pb2.ConfigValueType.Value("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>return proto_config<EOL>

Returns component-specific Config protobuf message

        It first adds ``topology.component.parallelism``, and is overriden by
        a user-defined component-specific configuration, specified by spec().


if not isinstance(custom_config, dict):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(custom_config)))<EOL><DEDENT>sanitized = {}<EOL>for key, value in custom_config.items():<EOL><INDENT>if not isinstance(key, str):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% (str(type(key)), str(key)))<EOL><DEDENT>if isinstance(value, bool):<EOL><INDENT>sanitized[key] = "<STR_LIT:true>" if value else "<STR_LIT:false>"<EOL><DEDENT>elif isinstance(value, (str, int, float)):<EOL><INDENT>sanitized[key] = str(value)<EOL><DEDENT>else:<EOL><INDENT>sanitized[key] = value<EOL><DEDENT><DEDENT>return sanitized<EOL>

Checks whether ``custom_config`` is sane and returns a sanitized dict <str -> (str|object)>

        It checks if keys are all strings and sanitizes values of a given dictionary as follows:

        - If string, number or boolean is given as a value, it is converted to string.
          For string and number (int, float), it is converted to string by a built-in ``str()`` method.
          For a boolean value, ``True`` is converted to "true" instead of "True", and ``False`` is
          converted to "false" instead of "False", in order to keep the consistency with
          Java configuration.

        - If neither of the above is given as a value, it is inserted into the sanitized dict as it is.
          These values will need to be serialized before adding to a protobuf message.


if self.inputs is None:<EOL><INDENT>return<EOL><DEDENT>input_dict = self._sanitize_inputs()<EOL>for global_streamid, gtype in input_dict.items():<EOL><INDENT>in_stream = bolt.inputs.add()<EOL>in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,<EOL>global_streamid.stream_id))<EOL>if isinstance(gtype, Grouping.FIELDS):<EOL><INDENT>in_stream.gtype = gtype.gtype<EOL>in_stream.grouping_fields.CopyFrom(self._get_stream_schema(gtype.fields))<EOL><DEDENT>elif isinstance(gtype, Grouping.CUSTOM):<EOL><INDENT>in_stream.gtype = gtype.gtype<EOL>in_stream.custom_grouping_object = gtype.python_serialized<EOL>in_stream.type = topology_pb2.CustomGroupingObjectType.Value("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>in_stream.gtype = gtype<EOL><DEDENT><DEDENT>

Adds inputs to a given protobuf Bolt message


ret = {}<EOL>if self.inputs is None:<EOL><INDENT>return<EOL><DEDENT>if isinstance(self.inputs, dict):<EOL><INDENT>for key, grouping in self.inputs.items():<EOL><INDENT>if not Grouping.is_grouping_sane(grouping):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if isinstance(key, HeronComponentSpec):<EOL><INDENT>if key.name is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>global_streamid = GlobalStreamId(key.name, Stream.DEFAULT_STREAM_ID)<EOL>ret[global_streamid] = grouping<EOL><DEDENT>elif isinstance(key, GlobalStreamId):<EOL><INDENT>ret[key] = grouping<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(key))<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(self.inputs, (list, tuple)):<EOL><INDENT>for input_obj in self.inputs:<EOL><INDENT>if isinstance(input_obj, HeronComponentSpec):<EOL><INDENT>if input_obj.name is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>global_streamid = GlobalStreamId(input_obj.name, Stream.DEFAULT_STREAM_ID)<EOL>ret[global_streamid] = Grouping.SHUFFLE<EOL><DEDENT>elif isinstance(input_obj, GlobalStreamId):<EOL><INDENT>ret[input_obj] = Grouping.SHUFFLE<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(input_obj))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % str(self.inputs))<EOL><DEDENT>return ret<EOL>

Sanitizes input fields and returns a map <GlobalStreamId -> Grouping>


if self.outputs is None:<EOL><INDENT>return<EOL><DEDENT>output_map = self._sanitize_outputs()<EOL>for stream_id, out_fields in output_map.items():<EOL><INDENT>out_stream = spbl.outputs.add()<EOL>out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))<EOL>out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))<EOL><DEDENT>

Adds outputs to a given protobuf Bolt or Spout message


ret = {}<EOL>if self.outputs is None:<EOL><INDENT>return<EOL><DEDENT>if not isinstance(self.outputs, (list, tuple)):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(self.outputs)))<EOL><DEDENT>for output in self.outputs:<EOL><INDENT>if not isinstance(output, (str, Stream)):<EOL><INDENT>raise TypeError("<STR_LIT>" % str(output))<EOL><DEDENT>if isinstance(output, str):<EOL><INDENT>if Stream.DEFAULT_STREAM_ID not in ret:<EOL><INDENT>ret[Stream.DEFAULT_STREAM_ID] = list()<EOL><DEDENT>ret[Stream.DEFAULT_STREAM_ID].append(output)<EOL><DEDENT>else:<EOL><INDENT>if output.stream_id == Stream.DEFAULT_STREAM_ID and Stream.DEFAULT_STREAM_ID in ret:<EOL><INDENT>ret[Stream.DEFAULT_STREAM_ID].extend(output.fields)<EOL><DEDENT>else:<EOL><INDENT>ret[output.stream_id] = output.fields<EOL><DEDENT><DEDENT><DEDENT>return ret<EOL>

Sanitizes output fields and returns a map <stream_id -> list of output fields>


if self.outputs is None:<EOL><INDENT>return set()<EOL><DEDENT>if not isinstance(self.outputs, (list, tuple)):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(self.outputs)))<EOL><DEDENT>ret_lst = []<EOL>for output in self.outputs:<EOL><INDENT>if not isinstance(output, (str, Stream)):<EOL><INDENT>raise TypeError("<STR_LIT>" % str(output))<EOL><DEDENT>ret_lst.append(Stream.DEFAULT_STREAM_ID if isinstance(output, str) else output.stream_id)<EOL><DEDENT>return set(ret_lst)<EOL>

Returns a set of output stream ids registered for this component


if stream_id not in self.get_out_streamids():<EOL><INDENT>raise ValueError("<STR_LIT>" % stream_id)<EOL><DEDENT>component_id = self.name or self<EOL>return GlobalStreamId(componentId=component_id, streamId=stream_id)<EOL>

Get GlobalStreamId for a given stream_id


proto_stream_id = topology_pb2.StreamId()<EOL>proto_stream_id.id = stream_id<EOL>proto_stream_id.component_name = comp_name<EOL>return proto_stream_id<EOL>


stream_schema = topology_pb2.StreamSchema()<EOL>for field in fields:<EOL><INDENT>key = stream_schema.keys.add()<EOL>key.key = field<EOL>key.type = topology_pb2.Type.Value("<STR_LIT>")<EOL><DEDENT>return stream_schema<EOL>


def __init__(self, componentId, streamId):

if not isinstance(componentId, (str, HeronComponentSpec)):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if not isinstance(streamId, str):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>self._component_id = componentId<EOL>self.stream_id = streamId<EOL>

:type componentId: str or HeronComponentSpec
:param componentId: component id from which the tuple is emitted, or HeronComponentSpec object.
:type streamId: str
:param streamId: stream id through which the tuple is transmitted


if isinstance(self._component_id, HeronComponentSpec):<EOL><INDENT>if self._component_id.name is None:<EOL><INDENT>return "<STR_LIT>" % self._component_id.uuid<EOL><DEDENT>return self._component_id.name<EOL><DEDENT>elif isinstance(self._component_id, str):<EOL><INDENT>return self._component_id<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>% (str(type(self._component_id)), str(self._component_id)))<EOL><DEDENT>

Returns component_id of this GlobalStreamId

        Note that if HeronComponentSpec is specified as componentId and its name is not yet
        available (i.e. when ``name`` argument was not given in ``spec()`` method in Bolt or Spout),
        this property returns a message with uuid. However, this is provided only for safety
        with __eq__(), __str__(), and __hash__() methods, and not meant to be called explicitly
        before TopologyType class finally sets the name attribute of HeronComponentSpec.


self.delegate = delegate<EOL>self.logger = self.delegate.logger<EOL>

Initializes BaseComponent

        :param delegate: SpoutInstance or BoltInstance


self.delegate.log(message, level)<EOL>

Log message, optionally providing a logging level

        :type message: str
        :param message: the log message to send
        :type level: str
        :param level: the logging level,
                      one of: trace (=debug), debug, info, warn or error (default: info)


@abstractmethod<EOL><INDENT>def prepare(self, conf, context):<DEDENT>

pass<EOL>

Called after the spout/bolt's initialize() method is called

        :param conf: component-specific configuration passed to the topology
        :param context: topology context


pass<EOL>

Called just before the spout/bolt's cleanup method is called


pass<EOL>

Called every time a tuple is emitted in spout/bolt

        :param emit_info: EmitInfo object


pass<EOL>

Called in spout every time a tuple gets acked

        :param spout_ack_info: SpoutAckInfo object


pass<EOL>

Called in spout every time a tuple gets failed

        :param spout_fail_info: SpoutFailInfo object


pass<EOL>

Called in bolt every time a tuple gets executed

        :param bolt_execute_info: BoltExecuteInfo object


pass<EOL>

Called in bolt every time a tuple gets acked

        :param bolt_ack_info: BoltAckInfo object


pass<EOL>

Called in bolt every time a tuple gets failed

        :param bolt_fail_info: BoltFailInfo object


@classmethod<EOL><INDENT>def spec(cls, name=None, par=<NUM_LIT:1>, config=None, optional_outputs=None):<DEDENT>

python_class_path = "<STR_LIT>" % (cls.__module__, cls.__name__)<EOL>if hasattr(cls, '<STR_LIT>'):<EOL><INDENT>_outputs = copy.copy(cls.outputs)<EOL><DEDENT>else:<EOL><INDENT>_outputs = []<EOL><DEDENT>if optional_outputs is not None:<EOL><INDENT>assert isinstance(optional_outputs, (list, tuple))<EOL>for out in optional_outputs:<EOL><INDENT>assert isinstance(out, (str, Stream))<EOL>_outputs.append(out)<EOL><DEDENT><DEDENT>return HeronComponentSpec(name, python_class_path, is_spout=True, par=par,<EOL>inputs=None, outputs=_outputs, config=config)<EOL>

Register this spout to the topology and create ``HeronComponentSpec``

        The usage of this method is compatible with StreamParse API, although it does not create
        ``ShellBoltSpec`` but instead directly registers to a ``Topology`` class.

        This method takes an optional ``outputs`` argument for supporting dynamic output fields
        declaration. However, it is recommended that ``outputs`` should be declared as
        an attribute of your ``Spout`` subclass. Also, some ways of declaring inputs is not supported
        in this implementation; please read the documentation below.

        :type name: str
        :param name: Name of this spout.
        :type par: int
        :param par: Parallelism hint for this spout.
        :type config: dict
        :param config: Component-specific config settings.
        :type optional_outputs: list of (str or Stream) or tuple of (str or Stream)
        :param optional_outputs: Additional output fields for this spout. These fields are added to
                                 existing ``outputs`` class attributes of your spout.
                                 This is an optional argument, and exists only for supporting dynamic
                                 output field declaration.


def emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID,<EOL>direct_task=None, need_task_ids=False):

return self.delegate.emit(tup, tup_id, stream, direct_task, need_task_ids)<EOL>

Emits a new tuple from this Spout

        It is compatible with StreamParse API.

        :type tup: list or tuple
        :param tup: the new output Tuple to send from this spout,
                    should contain only serializable data.
        :type tup_id: str or object
        :param tup_id: the ID for the Tuple. Leave this blank for an unreliable emit.
                       (Same as messageId in Java)
        :type stream: str
        :param stream: the ID of the stream this Tuple should be emitted to.
                       Leave empty to emit to the default stream.
        :type direct_task: int
        :param direct_task: the task to send the Tuple to if performing a direct emit.
        :type need_task_ids: bool
        :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.


pass<EOL>

Called when a task for this component is initialized within a worker on the cluster

        It provides the spout with the environment in which the spout executes. Note that
        you should NOT override ``__init__()`` for initialization of your spout, as it is
        used internally by Heron Instance; instead, you should use this method to initialize
        any custom instance variables or connections to data sources.

        *Should be implemented by a subclass.*

        :type config: dict
        :param config: The Heron configuration for this bolt. This is the configuration provided to
                       the topology merged in with cluster configuration on this machine.
                       Note that types of string values in the config have been automatically converted,
                       meaning that number strings and boolean strings are converted to appropriate
                       types.
        :type context: :class:`TopologyContext`
        :param context: This object can be used to get information about this task's place within the
                        topology, including the task id and component id of this task, input and output
                        information, etc.


pass<EOL>

Called when this spout is going to be shutdown

        There is no guarantee that close() will be called.


raise NotImplementedError("<STR_LIT>")<EOL>

When this method is called, Heron is requesting that the Spout emit tuples

        It is compatible with StreamParse API.

        This method should be non-blocking, so if the Spout has no tuples to emit,
        this method should return; next_tuple(), ack(), and fail() are all called in a tight
        loop in a single thread in the spout task. WHen there are no tuples to emit, it is
        courteous to have next_tuple sleep for a short amount of time (like a single millisecond)
        so as not to waste too much CPU.

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


pass<EOL>

Determine that the tuple emitted by this spout with the tup_id has been fully processed

        It is compatible with StreamParse API.

        Heron has determined that the tuple emitted by this spout with the tup_id identifier
        has been fully processed. Typically, an implementation of this method will take that
        message off the queue and prevent it from being replayed.

        *Should be implemented by a subclass.*

        :param tup_id: the ID of the HeronTuple that has been fully acknowledged.


pass<EOL>

Determine that the tuple emitted by this spout with the tup_id has failed to be processed

        It is compatible with StreamParse API.

        The tuple emitted by this spout with the tup_id identifier has failed to be
        fully processed. Typically, an implementation of this method will put that
        message back on the queue to be replayed at a later time.

        *Should be implemented by a subclass.*

        :param tup_id: the ID of the HeronTuple that has failed either due to a bolt calling ``fail()``
                       or timeout


pass<EOL>

Called when a spout has been activated out of a deactivated mode

        next_tuple() will be called on this spout soon. A spout can become activated
        after having been deactivated when the topology is manipulated using the
        `heron` client.


pass<EOL>

Called when a spout has been deactivated

        next_tuple() will not be called while a spout is deactivated.
        The spout may or may not be reactivated in the future.


def __init__(self, fields=None, name=DEFAULT_STREAM_ID, direct=False):

if fields is None:<EOL><INDENT>fields = []<EOL><DEDENT>elif isinstance(fields, (list, tuple)):<EOL><INDENT>fields = list(fields)<EOL>for field in fields:<EOL><INDENT>if not isinstance(field, str):<EOL><INDENT>raise TypeError("<STR_LIT>" % str(field))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % str(fields))<EOL><DEDENT>self.fields = fields<EOL>if name is None:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>elif isinstance(name, str):<EOL><INDENT>self.stream_id = name<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % str(name))<EOL><DEDENT>if isinstance(direct, bool):<EOL><INDENT>self.direct = direct<EOL>if self.direct:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % str(direct))<EOL><DEDENT>

:type fields: `list` or `tuple` of `str`
:param fields: field names for this stream
:type name: str
:param name: name of stream. Defaults to ``default``
:type direct: bool
:param direct: whether or not this stream is direct. Default is ``False``


if gtype == cls.SHUFFLE or gtype == cls.ALL or gtype == cls.LOWEST or gtype == cls.NONE:<EOL><INDENT>return True<EOL><DEDENT>elif isinstance(gtype, cls.FIELDS):<EOL><INDENT>return gtype.gtype == topology_pb2.Grouping.Value("<STR_LIT>") andgtype.fields is not None<EOL><DEDENT>elif isinstance(gtype, cls.CUSTOM):<EOL><INDENT>return gtype.gtype == topology_pb2.Grouping.Value("<STR_LIT>") andgtype.python_serialized is not None<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>


if len(fields) == <NUM_LIT:1> and isinstance(fields[<NUM_LIT:0>], list):<EOL><INDENT>fields = fields[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>fields = list(fields)<EOL><DEDENT>for i in fields:<EOL><INDENT>if not isinstance(i, str):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT><DEDENT>if not fields:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return cls.FIELDS(gtype=topology_pb2.Grouping.Value("<STR_LIT>"),<EOL>fields=fields)<EOL>


if customgrouper is None:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>if not isinstance(customgrouper, ICustomGrouping) and not isinstance(customgrouper, str):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>serialized = default_serializer.serialize(customgrouper)<EOL>return cls.custom_serialized(serialized, is_java=False)<EOL>

Custom grouping from a given implementation of ICustomGrouping

        :param customgrouper: The ICustomGrouping implemention to use


@classmethod<EOL><INDENT>def custom_serialized(cls, serialized, is_java=True):<DEDENT>

if not isinstance(serialized, bytes):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>" % str(serialized))<EOL><DEDENT>if not is_java:<EOL><INDENT>return cls.CUSTOM(gtype=topology_pb2.Grouping.Value("<STR_LIT>"),<EOL>python_serialized=serialized)<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Custom grouping from a given serialized string

        This class is created for compatibility with ``custom_serialized(cls, java_serialized)`` method
        of StreamParse API, although its functionality is not yet implemented (Java-serialized).
        Currently only custom grouping implemented in Python is supported, and ``custom()`` method
        should be used to indicate its classpath, rather than directly to use this method.

        In the future, users can directly specify Java-serialized object with ``is_java=True`` in order
        to use a custom grouping implemented in Java for python topology.

        :param serialized: serialized classpath to custom grouping class to use (if python)
        :param is_java: indicate whether this is Java serialized, or python serialized


@classmethod<EOL><INDENT>def custom_object(cls, java_class_name, arg_list):<DEDENT>

Tuples will be assigned to tasks by the given Java class.


pass<EOL>


pass<EOL>

Serialize an object

        :param obj: The object to be serialized
        :returns: Serialized object as byte string


pass<EOL>

Deserialize an object

        :param input_str: Serialized object as byte string
        :returns: Deserialized object


if not modList:<EOL><INDENT>return<EOL><DEDENT>main = sys.modules['<STR_LIT:__main__>']<EOL>for modname in modList:<EOL><INDENT>if isinstance(modname, str):<EOL><INDENT>try:<EOL><INDENT>mod = __import__(modname)<EOL><DEDENT>except Exception:<EOL><INDENT>sys.stderr.write(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % modname)<EOL>print_exec(sys.stderr)<EOL><DEDENT>else:<EOL><INDENT>setattr(main, mod.__name__, mod)<EOL><DEDENT><DEDENT><DEDENT>

Force every module in modList to be placed into main


def _fill_function(func, globalsn, defaults, dictn, module):

func.__globals__.update(globalsn)<EOL>func.__defaults__ = defaults<EOL>func.__dict__ = dictn<EOL>func.__module__ = module<EOL>return func<EOL>

Fills in the rest of function data into the skeleton function object
      that were created via _make_skel_func().


def _make_skel_func(code, closures, base_globals=None):

closure = _reconstruct_closure(closures) if closures else None<EOL>if base_globals is None:<EOL><INDENT>base_globals = {}<EOL><DEDENT>base_globals['<STR_LIT>'] = __builtins__<EOL>return types.FunctionType(code, base_globals, None, None, closure)<EOL>

Creates a skeleton function object that contains just the provided
      code and the correct number of cells in func_closure.  All other
      func attributes (e.g. func_globals) are empty.


for k, v in d.items():<EOL><INDENT>if isinstance(k, tuple):<EOL><INDENT>typ, k = k<EOL>if typ == '<STR_LIT>':<EOL><INDENT>v = property(*v)<EOL><DEDENT>elif typ == '<STR_LIT>':<EOL><INDENT>v = staticmethod(v) <EOL><DEDENT>elif typ == '<STR_LIT>':<EOL><INDENT>v = classmethod(v)<EOL><DEDENT><DEDENT>setattr(cls, k, v)<EOL><DEDENT>return cls<EOL>

Loads additional properties into class `cls`.


from collections import namedtuple<EOL>return namedtuple(name, fields)<EOL>


Pickler.save_string(self, str(obj))<EOL>


self.modules.add(obj)<EOL>self.save_reduce(subimport, (obj.__name__,), obj=obj)<EOL>


if PY3:<EOL><INDENT>args = (<EOL>obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,<EOL>obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames,<EOL>obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,<EOL>obj.co_cellvars<EOL>)<EOL><DEDENT>else:<EOL><INDENT>args = (<EOL>obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code,<EOL>obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name,<EOL>obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars<EOL>)<EOL><DEDENT>self.save_reduce(types.CodeType, args, obj=obj)<EOL>


def save_function(self, obj, name=None):

write = self.write<EOL>if name is None:<EOL><INDENT>name = obj.__name__<EOL><DEDENT>try:<EOL><INDENT>modname = pickle.whichmodule(obj, name)<EOL><DEDENT>except Exception:<EOL><INDENT>modname = None<EOL><DEDENT>try:<EOL><INDENT>themodule = sys.modules[modname]<EOL><DEDENT>except KeyError:<EOL><INDENT>modname = '<STR_LIT:__main__>'<EOL><DEDENT>if modname == '<STR_LIT:__main__>':<EOL><INDENT>themodule = None<EOL><DEDENT>if themodule:<EOL><INDENT>self.modules.add(themodule)<EOL>if getattr(themodule, name, None) is obj:<EOL><INDENT>return self.save_global(obj, name)<EOL><DEDENT><DEDENT>if islambda(obj) or obj.__code__.co_filename == '<STR_LIT>' or themodule is None:<EOL><INDENT>self.save_function_tuple(obj)<EOL>return<EOL><DEDENT>else:<EOL><INDENT>klass = getattr(themodule, name, None)<EOL>if klass is None or klass is not obj:<EOL><INDENT>self.save_function_tuple(obj)<EOL>return<EOL><DEDENT><DEDENT>if obj.__dict__:<EOL><INDENT>self.save(_restore_attr)<EOL>write(pickle.MARK + pickle.GLOBAL + modname + '<STR_LIT:\n>' + name + '<STR_LIT:\n>')<EOL>self.memoize(obj)<EOL>self.save(obj.__dict__)<EOL>write(pickle.TUPLE + pickle.REDUCE)<EOL><DEDENT>else:<EOL><INDENT>write(pickle.GLOBAL + modname + '<STR_LIT:\n>' + name + '<STR_LIT:\n>')<EOL>self.memoize(obj)<EOL><DEDENT>

Registered with the dispatch to handle all function types.
        Determines what kind of function obj is (e.g. lambda, defined at
        interactive prompt, etc) and handles the pickling appropriately.


save = self.save<EOL>write = self.write<EOL>code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data(func)<EOL>save(_fill_function)  <EOL>write(pickle.MARK)    <EOL>save(_make_skel_func)<EOL>save((code, closure, base_globals))<EOL>write(pickle.REDUCE)<EOL>self.memoize(func)<EOL>save(f_globals)<EOL>save(defaults)<EOL>save(dct)<EOL>save(func.__module__)<EOL>write(pickle.TUPLE)<EOL>write(pickle.REDUCE)<EOL>

Pickles an actual func object.
        A func comprises: code, globals, defaults, closure, and dict.  We
        extract and save these, injecting reducing functions at certain points
        to recreate the func object.  Keep in mind that some of these pieces
        can contain a ref to the func itself.  Thus, a naive save on these
        pieces could trigger an infinite loop of save's.  To get around that,
        we first create a skeleton func object using just the code (this is
        safe, since this won't contain a ref to the func), and memoize it as
        soon as it's created.  The other stuff can then be filled in later.


out_names = cls._extract_code_globals_cache.get(co)<EOL>if out_names is None:<EOL><INDENT>try:<EOL><INDENT>names = co.co_names<EOL><DEDENT>except AttributeError:<EOL><INDENT>out_names = set()<EOL><DEDENT>else:<EOL><INDENT>out_names = set(names[oparg]<EOL>for op, oparg in _walk_global_ops(co))<EOL>if co.co_consts:<EOL><INDENT>for const in co.co_consts:<EOL><INDENT>if type(const) is types.CodeType: <EOL><INDENT>out_names |= cls.extract_code_globals(const)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>cls._extract_code_globals_cache[co] = out_names<EOL><DEDENT>return out_names<EOL>

Find all globals names read or written to by codeblock co


code = func.__code__<EOL>func_global_refs = self.extract_code_globals(code)<EOL>f_globals = {}<EOL>for var in func_global_refs:<EOL><INDENT>if var in func.__globals__:<EOL><INDENT>f_globals[var] = func.__globals__[var]<EOL><DEDENT><DEDENT>defaults = func.__defaults__<EOL>closure = [c.cell_contents for c in func.__closure__] if func.__closure__ else []<EOL>dct = func.__dict__<EOL>base_globals = self.globals_ref.get(id(func.__globals__), {})<EOL>self.globals_ref[id(func.__globals__)] = base_globals<EOL>return (code, f_globals, defaults, closure, dct, base_globals)<EOL>

Turn the function into a tuple of data necessary to recreate it:
  code, globals, defaults, closure, dict


cls = obj.__class__<EOL>memo = self.memo<EOL>write = self.write<EOL>save = self.save<EOL>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>args = obj.__getinitargs__()<EOL>len(args)  <EOL>pickle._keep_alive(args, memo) <EOL><DEDENT>else:<EOL><INDENT>args = ()<EOL><DEDENT>write(pickle.MARK)<EOL>if self.bin:<EOL><INDENT>save(cls)<EOL>for arg in args:<EOL><INDENT>save(arg)<EOL><DEDENT>write(pickle.OBJ)<EOL><DEDENT>else:<EOL><INDENT>for arg in args:<EOL><INDENT>save(arg)<EOL><DEDENT>write(pickle.INST + cls.__module__ + '<STR_LIT:\n>' + cls.__name__ + '<STR_LIT:\n>')<EOL><DEDENT>self.memoize(obj)<EOL>try:<EOL><INDENT>getstate = obj.__getstate__<EOL><DEDENT>except AttributeError:<EOL><INDENT>stuff = obj.__dict__<EOL>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>transient = obj.__transient__<EOL>stuff = stuff.copy()<EOL>for k in list(stuff.keys()):<EOL><INDENT>if k in transient:<EOL><INDENT>del stuff[k]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>stuff = getstate()<EOL>pickle._keep_alive(stuff, memo) <EOL><DEDENT>save(stuff)<EOL>write(pickle.BUILD)<EOL>

Inner logic to save instance. Based off pickle.save_inst
        Supports __transient__


class Dummy: <EOL><INDENT>def __init__(self):<EOL><INDENT>pass<EOL><DEDENT>def __getitem__(self, item):<EOL><INDENT>return item<EOL><DEDENT><DEDENT>items = obj(Dummy())<EOL>if not isinstance(items, tuple):<EOL><INDENT>items = (items, )<EOL><DEDENT>return self.save_reduce(operator.itemgetter, items)<EOL>

itemgetter serializer (needed for namedtuple support)


class Dummy(object):<EOL><INDENT>def __init__(self, attrs, index=None):<EOL><INDENT>self.attrs = attrs<EOL>self.index = index<EOL><DEDENT>def __getattribute__(self, item):<EOL><INDENT>attrs = object.__getattribute__(self, "<STR_LIT>")<EOL>index = object.__getattribute__(self, "<STR_LIT:index>")<EOL>if index is None:<EOL><INDENT>index = len(attrs)<EOL>attrs.append(item)<EOL><DEDENT>else:<EOL><INDENT>attrs[index] = "<STR_LIT:.>".join([attrs[index], item])<EOL><DEDENT>return type(self)(attrs, index)<EOL><DEDENT><DEDENT>attrs = []<EOL>obj(Dummy(attrs))<EOL>return self.save_reduce(operator.attrgetter, tuple(attrs))<EOL>


def save_reduce(self, func, args, state=None, <EOL>listitems=None, dictitems=None, obj=None):

<EOL>if not isinstance(args, tuple):<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>")<EOL><DEDENT>if not hasattr(func, '<STR_LIT>'):<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>")<EOL><DEDENT>save = self.save<EOL>write = self.write<EOL>if self.proto >= <NUM_LIT:2> and getattr(func, "<STR_LIT>", "<STR_LIT>") == "<STR_LIT>":<EOL><INDENT>cls = args[<NUM_LIT:0>]<EOL>if not hasattr(cls, "<STR_LIT>"):<EOL><INDENT>raise pickle.PicklingError(<EOL>"<STR_LIT>")<EOL><DEDENT>if obj is not None and cls is not obj.__class__:<EOL><INDENT>raise pickle.PicklingError(<EOL>"<STR_LIT>")<EOL><DEDENT>args = args[<NUM_LIT:1>:]<EOL>save(cls)<EOL>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>transient = obj.__transient__<EOL>state = state.copy()<EOL>for k in list(state.keys()):<EOL><INDENT>if k in transient:<EOL><INDENT>del state[k]<EOL><DEDENT><DEDENT><DEDENT>save(args)<EOL>write(pickle.NEWOBJ)<EOL><DEDENT>else:<EOL><INDENT>save(func)<EOL>save(args)<EOL>write(pickle.REDUCE)<EOL><DEDENT>if obj is not None:<EOL><INDENT>self.memoize(obj)<EOL><DEDENT>if listitems is not None:<EOL><INDENT>self._batch_appends(listitems)<EOL><DEDENT>if dictitems is not None:<EOL><INDENT>self._batch_setitems(dictitems)<EOL><DEDENT>if state is not None:<EOL><INDENT>save(state)<EOL>write(pickle.BUILD)<EOL><DEDENT>

Modified to support __transient__ on new objects
        Change only affects protocol level 2 (which is always used by PiCloud


self.save_reduce(_genpartial, (obj.func, obj.args, obj.keywords))<EOL>

Partial objects do not serialize correctly in python2.x -- this fixes the bugs


try:<EOL><INDENT>import StringIO as pystringIO <EOL><DEDENT>except ImportError:<EOL><INDENT>import io as pystringIO <EOL><DEDENT>if not hasattr(obj, '<STR_LIT:name>') or  not hasattr(obj, '<STR_LIT>'):<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>")<EOL><DEDENT>if obj is sys.stdout:<EOL><INDENT>return self.save_reduce(getattr, (sys, '<STR_LIT>'), obj=obj)<EOL><DEDENT>if obj is sys.stderr:<EOL><INDENT>return self.save_reduce(getattr, (sys, '<STR_LIT>'), obj=obj)<EOL><DEDENT>if obj is sys.stdin:<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>")<EOL><DEDENT>if  hasattr(obj, '<STR_LIT>') and obj.isatty():<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>")<EOL><DEDENT>if '<STR_LIT:r>' not in obj.mode:<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>")<EOL><DEDENT>name = obj.name<EOL>try:<EOL><INDENT>fsize = os.stat(name).st_size<EOL><DEDENT>except OSError:<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>" % name)<EOL><DEDENT>if obj.closed:<EOL><INDENT>retval = pystringIO.StringIO("<STR_LIT>")<EOL>retval.close()<EOL><DEDENT>elif not fsize: <EOL><INDENT>retval = pystringIO.StringIO("<STR_LIT>")<EOL>try:<EOL><INDENT>tmpfile = file(name)<EOL>tst = tmpfile.read(<NUM_LIT:1>)<EOL><DEDENT>except IOError:<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>" % name)<EOL><DEDENT>tmpfile.close()<EOL>if tst != '<STR_LIT>':<EOL><INDENT>raise pickle.PicklingError(<EOL>"<STR_LIT>" % name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>tmpfile = file(name)<EOL>contents = tmpfile.read()<EOL>tmpfile.close()<EOL><DEDENT>except IOError:<EOL><INDENT>raise pickle.PicklingError("<STR_LIT>" % name)<EOL><DEDENT>retval = pystringIO.StringIO(contents)<EOL>curloc = obj.tell()<EOL>retval.seek(curloc)<EOL><DEDENT>retval.name = name<EOL>self.save(retval)<EOL>self.memoize(obj)<EOL>


name = obj.__name__<EOL>numpy_tst_mods = ['<STR_LIT>', '<STR_LIT>']<EOL>for tst_mod_name in numpy_tst_mods:<EOL><INDENT>tst_mod = sys.modules.get(tst_mod_name, None)<EOL>if tst_mod and name in tst_mod.__dict__:<EOL><INDENT>return self.save_reduce(_getobject, (tst_mod_name, name))<EOL><DEDENT><DEDENT>raise pickle.PicklingError(<EOL>'<STR_LIT>' % str(obj))<EOL>

Hack function for saving numpy ufunc objects


Plug in system. Register additional pickling functions if modules already loaded


def ParseNolintSuppressions(filename, raw_line, linenum, error):

matched = Search(r'<STR_LIT>', raw_line)<EOL>if matched:<EOL><INDENT>if matched.group(<NUM_LIT:1>):<EOL><INDENT>suppressed_line = linenum + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>suppressed_line = linenum<EOL><DEDENT>category = matched.group(<NUM_LIT:2>)<EOL>if category in (None, '<STR_LIT>'):  <EOL><INDENT>_error_suppressions.setdefault(None, set()).add(suppressed_line)<EOL><DEDENT>else:<EOL><INDENT>if category.startswith('<STR_LIT:(>') and category.endswith('<STR_LIT:)>'):<EOL><INDENT>category = category[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>if category in _ERROR_CATEGORIES:<EOL><INDENT>_error_suppressions.setdefault(category, set()).add(suppressed_line)<EOL><DEDENT>elif category not in _LEGACY_ERROR_CATEGORIES:<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' % category)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Updates the global list of line error-suppressions.

    Parses any NOLINT comments on the current line, updating the global
    error_suppressions store.  Reports an error if the NOLINT comment
    was malformed.

    Args:
      filename: str, the name of the input file.
      raw_line: str, the line of input text, with comments.
      linenum: int, the number of the current line.
      error: function, an error handler.


for line in lines:<EOL><INDENT>if _SEARCH_C_FILE.search(line):<EOL><INDENT>for category in _DEFAULT_C_SUPPRESSED_CATEGORIES:<EOL><INDENT>_global_error_suppressions[category] = True<EOL><DEDENT><DEDENT>if _SEARCH_KERNEL_FILE.search(line):<EOL><INDENT>for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES:<EOL><INDENT>_global_error_suppressions[category] = True<EOL><DEDENT><DEDENT><DEDENT>

Updates the list of global error suppressions.

    Parses any lint directives in the file that have global effect.

    Args:
      lines: An array of strings, each representing a line of the file, with the
             last element being empty if the file is terminated with a newline.


_error_suppressions.clear()<EOL>_global_error_suppressions.clear()<EOL>

Resets the set of NOLINT suppressions to empty.


return (_global_error_suppressions.get(category, False) or<EOL>linenum in _error_suppressions.get(category, set()) or<EOL>linenum in _error_suppressions.get(None, set()))<EOL>

Returns true if the specified error category is suppressed on this line.

    Consults the global error_suppressions map populated by
    ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions.

    Args:
      category: str, the category of the error.
      linenum: int, the current line number.
    Returns:
      bool, True iff the error should be suppressed due to a NOLINT comment or
      global suppression.


<EOL>if pattern not in _regexp_compile_cache:<EOL><INDENT>_regexp_compile_cache[pattern] = sre_compile.compile(pattern)<EOL><DEDENT>return _regexp_compile_cache[pattern].match(s)<EOL>

Matches the string with the pattern, caching the compiled regexp.


if pattern not in _regexp_compile_cache:<EOL><INDENT>_regexp_compile_cache[pattern] = sre_compile.compile(pattern)<EOL><DEDENT>return _regexp_compile_cache[pattern].sub(rep, s)<EOL>

Replaces instances of pattern in a string with a replacement.

    The compiled regex is kept in a cache shared by Match and Search.

    Args:
      pattern: regex pattern
      rep: replacement text
      s: search string

    Returns:
      string with replacements made (or original string if no replacements)


if pattern not in _regexp_compile_cache:<EOL><INDENT>_regexp_compile_cache[pattern] = sre_compile.compile(pattern)<EOL><DEDENT>return _regexp_compile_cache[pattern].search(s)<EOL>

Searches the string for the pattern, caching the compiled regexp.


return s in GetNonHeaderExtensions()<EOL>

File extension (excluding dot) matches a source file extension.


return _cpplint_state.output_format<EOL>


_cpplint_state.SetOutputFormat(output_format)<EOL>


return _cpplint_state.verbose_level<EOL>


return _cpplint_state.SetVerboseLevel(level)<EOL>

Sets the module's verbosity, and returns the previous setting.


_cpplint_state.SetCountingStyle(level)<EOL>


return _cpplint_state.filters<EOL>

Returns the module's list of output filters, as a list.


Sets the module's error-message filters.

    These filters are applied when deciding whether to emit a given
    error message.

    Args:
      filters: A string of comma-separated filters (eg "whitespace/indent").
               Each filter should start with + or -; else we die.


Adds more filter overrides.

    Unlike _SetFilters, this function does not reset the current list of filters
    available.

    Args:
      filters: A string of comma-separated filters (eg "whitespace/indent").
               Each filter should start with + or -; else we die.


Saves the current filter list to backup storage.


def _ShouldPrintError(category, confidence, linenum):

<EOL>if IsErrorSuppressedByNolint(category, linenum):<EOL><INDENT>return False<EOL><DEDENT>if confidence < _cpplint_state.verbose_level:<EOL><INDENT>return False<EOL><DEDENT>is_filtered = False<EOL>for one_filter in _Filters():<EOL><INDENT>if one_filter.startswith('<STR_LIT:->'):<EOL><INDENT>if category.startswith(one_filter[<NUM_LIT:1>:]):<EOL><INDENT>is_filtered = True<EOL><DEDENT><DEDENT>elif one_filter.startswith('<STR_LIT:+>'):<EOL><INDENT>if category.startswith(one_filter[<NUM_LIT:1>:]):<EOL><INDENT>is_filtered = False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert False  <EOL><DEDENT><DEDENT>if is_filtered:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

If confidence >= verbose, category passes filter and is not suppressed.


def Error(filename, linenum, category, confidence, message):

if _ShouldPrintError(category, confidence, linenum):<EOL><INDENT>_cpplint_state.IncrementErrorCount(category)<EOL>if _cpplint_state.output_format == '<STR_LIT>':<EOL><INDENT>_cpplint_state.PrintError('<STR_LIT>' % (<EOL>filename, linenum, message, category, confidence))<EOL><DEDENT>elif _cpplint_state.output_format == '<STR_LIT>':<EOL><INDENT>sys.stderr.write('<STR_LIT>' % (<EOL>filename, linenum, message, category, confidence))<EOL><DEDENT>elif _cpplint_state.output_format == '<STR_LIT>':<EOL><INDENT>_cpplint_state.AddJUnitFailure(filename, linenum, message, category,<EOL>confidence)<EOL><DEDENT>else:<EOL><INDENT>final_message = '<STR_LIT>' % (<EOL>filename, linenum, message, category, confidence)<EOL>sys.stderr.write(final_message)<EOL><DEDENT><DEDENT>

Logs the fact we've found a lint error.

    We log where the error was found, and also our confidence in the error,
    that is, how certain we are this is a legitimate style regression, and
    not a misidentification or a use that's sometimes justified.

    False positives can be suppressed by the use of
    "cpplint(category)"  comments on the offending line.  These are
    parsed into _error_suppressions.

    Args:
      filename: The name of the file containing the error.
      linenum: The number of the line containing the error.
      category: A string used to describe the "category" this bug
        falls under: "whitespace", say, or "runtime".  Categories
        may have a hierarchy separated by slashes: "whitespace/indent".
      confidence: A number from 1-5 representing a confidence score for
        the error, with 5 meaning that we are certain of the problem,
        and 1 meaning that it could be a legitimate construct.
      message: The error message.


line = line.replace(r'<STR_LIT:\\>', '<STR_LIT>')  <EOL>return ((line.count('<STR_LIT:">') - line.count(r'<STR_LIT>') - line.count("<STR_LIT>")) & <NUM_LIT:1>) == <NUM_LIT:1><EOL>

Does line terminate so, that the next symbol is in string constant.

    This function does not consider single-line nor multi-line comments.

    Args:
      line: is a partial line of code starting from the 0..n.

    Returns:
      True, if next character appended to 'line' is inside a
      string constant.


delimiter = None<EOL>lines_without_raw_strings = []<EOL>for line in raw_lines:<EOL><INDENT>if delimiter:<EOL><INDENT>end = line.find(delimiter)<EOL>if end >= <NUM_LIT:0>:<EOL><INDENT>leading_space = Match(r'<STR_LIT>', line)<EOL>line = leading_space.group(<NUM_LIT:1>) + '<STR_LIT>' + line[end + len(delimiter):]<EOL>delimiter = None<EOL><DEDENT>else:<EOL><INDENT>line = '<STR_LIT>'<EOL><DEDENT><DEDENT>while delimiter is None:<EOL><INDENT>matched = Match(r'<STR_LIT>', line)<EOL>if (matched and<EOL>not Match(r'<STR_LIT>',<EOL>matched.group(<NUM_LIT:1>))):<EOL><INDENT>delimiter = '<STR_LIT:)>' + matched.group(<NUM_LIT:2>) + '<STR_LIT:">'<EOL>end = matched.group(<NUM_LIT:3>).find(delimiter)<EOL>if end >= <NUM_LIT:0>:<EOL><INDENT>line = (matched.group(<NUM_LIT:1>) + '<STR_LIT>' +<EOL>matched.group(<NUM_LIT:3>)[end + len(delimiter):])<EOL>delimiter = None<EOL><DEDENT>else:<EOL><INDENT>line = matched.group(<NUM_LIT:1>) + '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>lines_without_raw_strings.append(line)<EOL><DEDENT>return lines_without_raw_strings<EOL>

Removes C++11 raw strings from lines.

      Before:
        static const char kData[] = R"(
            multi-line string
            )";

      After:
        static const char kData[] = ""
            (replaced by blank line)
            "";

    Args:
      raw_lines: list of raw lines.

    Returns:
      list of lines with C++11 raw strings replaced by empty strings.


while lineix < len(lines):<EOL><INDENT>if lines[lineix].strip().startswith('<STR_LIT>'):<EOL><INDENT>if lines[lineix].strip().find('<STR_LIT>', <NUM_LIT:2>) < <NUM_LIT:0>:<EOL><INDENT>return lineix<EOL><DEDENT><DEDENT>lineix += <NUM_LIT:1><EOL><DEDENT>return len(lines)<EOL>

Find the beginning marker for a multiline comment.


while lineix < len(lines):<EOL><INDENT>if lines[lineix].strip().endswith('<STR_LIT>'):<EOL><INDENT>return lineix<EOL><DEDENT>lineix += <NUM_LIT:1><EOL><DEDENT>return len(lines)<EOL>

We are inside a comment, find the end marker.


def RemoveMultiLineCommentsFromRange(lines, begin, end):

<EOL>for i in range(begin, end):<EOL><INDENT>lines[i] = '<STR_LIT>'<EOL><DEDENT>

Clears a range of lines for multi-line comments.


def RemoveMultiLineComments(filename, lines, error):

lineix = <NUM_LIT:0><EOL>while lineix < len(lines):<EOL><INDENT>lineix_begin = FindNextMultiLineCommentStart(lines, lineix)<EOL>if lineix_begin >= len(lines):<EOL><INDENT>return<EOL><DEDENT>lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin)<EOL>if lineix_end >= len(lines):<EOL><INDENT>error(filename, lineix_begin + <NUM_LIT:1>, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>')<EOL>return<EOL><DEDENT>RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + <NUM_LIT:1>)<EOL>lineix = lineix_end + <NUM_LIT:1><EOL><DEDENT>

Removes multiline (c-style) comments from lines.


commentpos = line.find('<STR_LIT>')<EOL>if commentpos != -<NUM_LIT:1> and not IsCppString(line[:commentpos]):<EOL><INDENT>line = line[:commentpos].rstrip()<EOL><DEDENT>return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('<STR_LIT>', line)<EOL>

Removes //-comments and single-line C-style /* */ comments.

    Args:
      line: A line of C++ source.

    Returns:
      The line with single-line comments removed.


def FindEndOfExpressionInLine(line, startpos, stack):

for i in xrange(startpos, len(line)):<EOL><INDENT>char = line[i]<EOL>if char in '<STR_LIT>':<EOL><INDENT>stack.append(char)<EOL><DEDENT>elif char == '<STR_LIT:<>':<EOL><INDENT>if i > <NUM_LIT:0> and line[i - <NUM_LIT:1>] == '<STR_LIT:<>':<EOL><INDENT>if stack and stack[-<NUM_LIT:1>] == '<STR_LIT:<>':<EOL><INDENT>stack.pop()<EOL>if not stack:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT><DEDENT><DEDENT>elif i > <NUM_LIT:0> and Search(r'<STR_LIT>', line[<NUM_LIT:0>:i]):<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>stack.append('<STR_LIT:<>')<EOL><DEDENT><DEDENT>elif char in '<STR_LIT>':<EOL><INDENT>while stack and stack[-<NUM_LIT:1>] == '<STR_LIT:<>':<EOL><INDENT>stack.pop()<EOL><DEDENT>if not stack:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT>if ((stack[-<NUM_LIT:1>] == '<STR_LIT:(>' and char == '<STR_LIT:)>') or<EOL>(stack[-<NUM_LIT:1>] == '<STR_LIT:[>' and char == '<STR_LIT:]>') or<EOL>(stack[-<NUM_LIT:1>] == '<STR_LIT:{>' and char == '<STR_LIT:}>')):<EOL><INDENT>stack.pop()<EOL>if not stack:<EOL><INDENT>return (i + <NUM_LIT:1>, None)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT><DEDENT>elif char == '<STR_LIT:>>':<EOL><INDENT>if (i > <NUM_LIT:0> and<EOL>(line[i - <NUM_LIT:1>] == '<STR_LIT:->' or Search(r'<STR_LIT>', line[<NUM_LIT:0>:i - <NUM_LIT:1>]))):<EOL><INDENT>continue<EOL><DEDENT>if stack:<EOL><INDENT>if stack[-<NUM_LIT:1>] == '<STR_LIT:<>':<EOL><INDENT>stack.pop()<EOL>if not stack:<EOL><INDENT>return (i + <NUM_LIT:1>, None)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif char == '<STR_LIT:;>':<EOL><INDENT>while stack and stack[-<NUM_LIT:1>] == '<STR_LIT:<>':<EOL><INDENT>stack.pop()<EOL><DEDENT>if not stack:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT><DEDENT><DEDENT>return (-<NUM_LIT:1>, stack)<EOL>

Find the position just after the end of current parenthesized expression.

    Args:
      line: a CleansedLines line.
      startpos: start searching at this position.
      stack: nesting stack at startpos.

    Returns:
      On finding matching end: (index just after matching end, None)
      On finding an unclosed expression: (-1, None)
      Otherwise: (-1, new stack at end of this line)


def CloseExpression(clean_lines, linenum, pos):

line = clean_lines.elided[linenum]<EOL>if (line[pos] not in '<STR_LIT>') or Match(r'<STR_LIT>', line[pos:]):<EOL><INDENT>return (line, clean_lines.NumLines(), -<NUM_LIT:1>)<EOL><DEDENT>(end_pos, stack) = FindEndOfExpressionInLine(line, pos, [])<EOL>if end_pos > -<NUM_LIT:1>:<EOL><INDENT>return (line, linenum, end_pos)<EOL><DEDENT>while stack and linenum < clean_lines.NumLines() - <NUM_LIT:1>:<EOL><INDENT>linenum += <NUM_LIT:1><EOL>line = clean_lines.elided[linenum]<EOL>(end_pos, stack) = FindEndOfExpressionInLine(line, <NUM_LIT:0>, stack)<EOL>if end_pos > -<NUM_LIT:1>:<EOL><INDENT>return (line, linenum, end_pos)<EOL><DEDENT><DEDENT>return (line, clean_lines.NumLines(), -<NUM_LIT:1>)<EOL>

If input points to ( or { or [ or <, finds the position that closes it.

    If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the
    linenum/pos that correspond to the closing of the expression.

    TODO(unknown): cpplint spends a fair bit of time matching parentheses.
    Ideally we would want to index all opening and closing parentheses once
    and have CloseExpression be just a simple lookup, but due to preprocessor
    tricks, this is not so easy.

    Args:
      clean_lines: A CleansedLines instance containing the file.
      linenum: The number of the line to check.
      pos: A position on the line.

    Returns:
      A tuple (line, linenum, pos) pointer *past* the closing brace, or
      (line, len(lines), -1) if we never find a close.  Note we ignore
      strings and comments when matching; and the line we return is the
      'cleansed' line at linenum.


def FindStartOfExpressionInLine(line, endpos, stack):

i = endpos<EOL>while i >= <NUM_LIT:0>:<EOL><INDENT>char = line[i]<EOL>if char in '<STR_LIT>':<EOL><INDENT>stack.append(char)<EOL><DEDENT>elif char == '<STR_LIT:>>':<EOL><INDENT>if (i > <NUM_LIT:0> and<EOL>(line[i - <NUM_LIT:1>] == '<STR_LIT:->' or<EOL>Match(r'<STR_LIT>', line[i - <NUM_LIT:1>:]) or<EOL>Search(r'<STR_LIT>', line[<NUM_LIT:0>:i]))):<EOL><INDENT>i -= <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>stack.append('<STR_LIT:>>')<EOL><DEDENT><DEDENT>elif char == '<STR_LIT:<>':<EOL><INDENT>if i > <NUM_LIT:0> and line[i - <NUM_LIT:1>] == '<STR_LIT:<>':<EOL><INDENT>i -= <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>if stack and stack[-<NUM_LIT:1>] == '<STR_LIT:>>':<EOL><INDENT>stack.pop()<EOL>if not stack:<EOL><INDENT>return (i, None)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif char in '<STR_LIT>':<EOL><INDENT>while stack and stack[-<NUM_LIT:1>] == '<STR_LIT:>>':<EOL><INDENT>stack.pop()<EOL><DEDENT>if not stack:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT>if ((char == '<STR_LIT:(>' and stack[-<NUM_LIT:1>] == '<STR_LIT:)>') or<EOL>(char == '<STR_LIT:[>' and stack[-<NUM_LIT:1>] == '<STR_LIT:]>') or<EOL>(char == '<STR_LIT:{>' and stack[-<NUM_LIT:1>] == '<STR_LIT:}>')):<EOL><INDENT>stack.pop()<EOL>if not stack:<EOL><INDENT>return (i, None)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT><DEDENT>elif char == '<STR_LIT:;>':<EOL><INDENT>while stack and stack[-<NUM_LIT:1>] == '<STR_LIT:>>':<EOL><INDENT>stack.pop()<EOL><DEDENT>if not stack:<EOL><INDENT>return (-<NUM_LIT:1>, None)<EOL><DEDENT><DEDENT>i -= <NUM_LIT:1><EOL><DEDENT>return (-<NUM_LIT:1>, stack)<EOL>

Find position at the matching start of current expression.

    This is almost the reverse of FindEndOfExpressionInLine, but note
    that the input position and returned position differs by 1.

    Args:
      line: a CleansedLines line.
      endpos: start searching at this position.
      stack: nesting stack at endpos.

    Returns:
      On finding matching start: (index at matching start, None)
      On finding an unclosed expression: (-1, None)
      Otherwise: (-1, new stack at beginning of this line)


def ReverseCloseExpression(clean_lines, linenum, pos):

line = clean_lines.elided[linenum]<EOL>if line[pos] not in '<STR_LIT>':<EOL><INDENT>return (line, <NUM_LIT:0>, -<NUM_LIT:1>)<EOL><DEDENT>(start_pos, stack) = FindStartOfExpressionInLine(line, pos, [])<EOL>if start_pos > -<NUM_LIT:1>:<EOL><INDENT>return (line, linenum, start_pos)<EOL><DEDENT>while stack and linenum > <NUM_LIT:0>:<EOL><INDENT>linenum -= <NUM_LIT:1><EOL>line = clean_lines.elided[linenum]<EOL>(start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - <NUM_LIT:1>, stack)<EOL>if start_pos > -<NUM_LIT:1>:<EOL><INDENT>return (line, linenum, start_pos)<EOL><DEDENT><DEDENT>return (line, <NUM_LIT:0>, -<NUM_LIT:1>)<EOL>

If input points to ) or } or ] or >, finds the position that opens it.

    If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the
    linenum/pos that correspond to the opening of the expression.

    Args:
      clean_lines: A CleansedLines instance containing the file.
      linenum: The number of the line to check.
      pos: A position on the line.

    Returns:
      A tuple (line, linenum, pos) pointer *at* the opening brace, or
      (line, 0, -1) if we never find the matching opening brace.  Note
      we ignore strings and comments when matching; and the line we
      return is the 'cleansed' line at linenum.


def CheckForCopyright(filename, lines, error):

<EOL>for line in range(<NUM_LIT:1>, min(len(lines), <NUM_LIT:11>)):<EOL><INDENT>if re.search(r'<STR_LIT>', lines[line], re.I): break<EOL><DEDENT>else:                       <EOL><INDENT>error(filename, <NUM_LIT:0>, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>

Logs an error if no Copyright message appears at the top of the file.


indent = Match(r'<STR_LIT>', line)<EOL>if indent:<EOL><INDENT>return len(indent.group(<NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>

Return the number of leading spaces in line.

    Args:
      line: A string to check.

    Returns:
      An integer count of leading spaces, possibly zero.


<EOL>filename = re.sub(r'<STR_LIT>', '<STR_LIT>', filename)<EOL>filename = re.sub(r'<STR_LIT>', r'<STR_LIT>', filename)<EOL>filename = filename.replace('<STR_LIT>', '<STR_LIT>').replace('<STR_LIT>', '<STR_LIT>')<EOL>fileinfo = FileInfo(filename)<EOL>file_path_from_root = fileinfo.RepositoryName()<EOL>if _root:<EOL><INDENT>suffix = os.sep<EOL>if suffix == '<STR_LIT:\\>':<EOL><INDENT>suffix += '<STR_LIT:\\>'<EOL><DEDENT>file_path_from_root = re.sub('<STR_LIT>' + _root + suffix, '<STR_LIT>', file_path_from_root)<EOL><DEDENT>return re.sub(r'<STR_LIT>', '<STR_LIT:_>', file_path_from_root).upper() + '<STR_LIT:_>'<EOL>

Returns the CPP variable that should be used as a header guard.

    Args:
      filename: The name of a C++ header file.

    Returns:
      The CPP variable that should be used as a header guard in the
      named file.


def CheckForHeaderGuard(filename, clean_lines, error):

<EOL>raw_lines = clean_lines.lines_without_raw_strings<EOL>for i in raw_lines:<EOL><INDENT>if Search(r'<STR_LIT>', i):<EOL><INDENT>return<EOL><DEDENT><DEDENT>for i in raw_lines:<EOL><INDENT>if Search(r'<STR_LIT>', i):<EOL><INDENT>return<EOL><DEDENT><DEDENT>cppvar = GetHeaderGuardCPPVariable(filename)<EOL>ifndef = '<STR_LIT>'<EOL>ifndef_linenum = <NUM_LIT:0><EOL>define = '<STR_LIT>'<EOL>endif = '<STR_LIT>'<EOL>endif_linenum = <NUM_LIT:0><EOL>for linenum, line in enumerate(raw_lines):<EOL><INDENT>linesplit = line.split()<EOL>if len(linesplit) >= <NUM_LIT:2>:<EOL><INDENT>if not ifndef and linesplit[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>ifndef = linesplit[<NUM_LIT:1>]<EOL>ifndef_linenum = linenum<EOL><DEDENT>if not define and linesplit[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>define = linesplit[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>endif = line<EOL>endif_linenum = linenum<EOL><DEDENT><DEDENT>if not ifndef or not define or ifndef != define:<EOL><INDENT>error(filename, <NUM_LIT:0>, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' %<EOL>cppvar)<EOL>return<EOL><DEDENT>if ifndef != cppvar:<EOL><INDENT>error_level = <NUM_LIT:0><EOL>if ifndef != cppvar + '<STR_LIT:_>':<EOL><INDENT>error_level = <NUM_LIT:5><EOL><DEDENT>ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum,<EOL>error)<EOL>error(filename, ifndef_linenum, '<STR_LIT>', error_level,<EOL>'<STR_LIT>' % cppvar)<EOL><DEDENT>ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum,<EOL>error)<EOL>match = Match(r'<STR_LIT>' + cppvar + r'<STR_LIT>', endif)<EOL>if match:<EOL><INDENT>if match.group(<NUM_LIT:1>) == '<STR_LIT:_>':<EOL><INDENT>error(filename, endif_linenum, '<STR_LIT>', <NUM_LIT:0>,<EOL>'<STR_LIT>' % cppvar)<EOL><DEDENT>return<EOL><DEDENT>no_single_line_comments = True<EOL>for i in xrange(<NUM_LIT:1>, len(raw_lines) - <NUM_LIT:1>):<EOL><INDENT>line = raw_lines[i]<EOL>if Match(r'<STR_LIT>', line):<EOL><INDENT>no_single_line_comments = False<EOL>break<EOL><DEDENT><DEDENT>if no_single_line_comments:<EOL><INDENT>match = Match(r'<STR_LIT>' + cppvar + r'<STR_LIT>', endif)<EOL>if match:<EOL><INDENT>if match.group(<NUM_LIT:1>) == '<STR_LIT:_>':<EOL><INDENT>error(filename, endif_linenum, '<STR_LIT>', <NUM_LIT:0>,<EOL>'<STR_LIT>' % cppvar)<EOL><DEDENT>return<EOL><DEDENT><DEDENT>error(filename, endif_linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' % cppvar)<EOL>

Checks that the file contains a header guard.

    Logs an error if no #ifndef header guard is present.  For other
    headers, checks that the full pathname is used.

    Args:
      filename: The name of the C++ header file.
      clean_lines: A CleansedLines instance containing the file.
      error: The function to call with any errors found.


def CheckHeaderFileIncluded(filename, include_state, error):

<EOL>fileinfo = FileInfo(filename)<EOL>if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()):<EOL><INDENT>return<EOL><DEDENT>for ext in GetHeaderExtensions():<EOL><INDENT>basefilename = filename[<NUM_LIT:0>:len(filename) - len(fileinfo.Extension())]<EOL>headerfile = basefilename + '<STR_LIT:.>' + ext<EOL>if not os.path.exists(headerfile):<EOL><INDENT>continue<EOL><DEDENT>headername = FileInfo(headerfile).RepositoryName()<EOL>first_include = None<EOL>for section_list in include_state.include_list:<EOL><INDENT>for f in section_list:<EOL><INDENT>if headername in f[<NUM_LIT:0>] or f[<NUM_LIT:0>] in headername:<EOL><INDENT>return<EOL><DEDENT>if not first_include:<EOL><INDENT>first_include = f[<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT>error(filename, first_include, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>' % (fileinfo.RepositoryName(),<EOL>headername))<EOL><DEDENT>

Logs an error if a source file does not include its header.


def CheckForBadCharacters(filename, lines, error):

for linenum, line in enumerate(lines):<EOL><INDENT>if unicode_escape_decode('<STR_LIT>') in line:<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in line:<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>, '<STR_LIT>')<EOL><DEDENT><DEDENT>

Logs an error for each line containing bad characters.

    Two kinds of bad characters:

    1. Unicode replacement characters: These indicate that either the file
    contained invalid UTF-8 (likely) or Unicode replacement characters (which
    it shouldn't).  Note that it's possible for this to throw off line
    numbering if the invalid UTF-8 occurred adjacent to a newline.

    2. NUL bytes.  These are problematic for some tools.

    Args:
      filename: The name of the current file.
      lines: An array of strings, each representing a line of the file.
      error: The function to call with any errors found.


def CheckForNewlineAtEOF(filename, lines, error):

<EOL>if len(lines) < <NUM_LIT:3> or lines[-<NUM_LIT:2>]:<EOL><INDENT>error(filename, len(lines) - <NUM_LIT:2>, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>')<EOL><DEDENT>

Logs an error if there is no newline char at the end of the file.

    Args:
      filename: The name of the current file.
      lines: An array of strings, each representing a line of the file.
      error: The function to call with any errors found.


def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error):

line = clean_lines.elided[linenum]<EOL>line = line.replace('<STR_LIT>', '<STR_LIT>')<EOL>if line.count('<STR_LIT>') > line.count('<STR_LIT>'):<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if (line.count('<STR_LIT:">') - line.count('<STR_LIT>')) % <NUM_LIT:2>:<EOL><INDENT>error(filename, linenum, '<STR_LIT>', <NUM_LIT:5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>

Logs an error if we see /* ... */ or "..." that extend past one line.

    /* ... */ comments are legit inside macros, for one line.
    Otherwise, we prefer // comments, so it's ok to warn about the
    other.  Likewise, it's ok for strings to extend across multiple
    lines, as long as a line continuation character (backslash)
    terminates each line. Although not currently prohibited by the C++
    style guide, it's ugly and unnecessary. We don't do well with either
    in this lint program, so we warn about both.

    Args:
      filename: The name of the current file.
      clean_lines: A CleansedLines instance containing the file.
      linenum: The number of the line to check.
      error: The function to call with any errors found.