<EOL>if self.encoderType in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if '<STR_LIT>' in encoderSpec:<EOL><INDENT>self.minval = encoderSpec.pop('<STR_LIT>')<EOL><DEDENT>else: self.minval=None<EOL>if '<STR_LIT>' in encoderSpec:<EOL><INDENT>self.maxval = encoderSpec.pop('<STR_LIT>')<EOL><DEDENT>else: self.maxval = None<EOL>self.encoder=adaptive_scalar.AdaptiveScalarEncoder(name='<STR_LIT>',w=self.w, n=self.n, minval=self.minval, maxval=self.maxval, periodic=False, forced=True)<EOL><DEDENT>elif self.encoderType=='<STR_LIT>':<EOL><INDENT>self.encoder=sdr_category.SDRCategoryEncoder(name='<STR_LIT>',w=self.w, n=self.n)<EOL><DEDENT>elif self.encoderType in ['<STR_LIT:date>', '<STR_LIT>']:<EOL><INDENT>self.encoder=date.DateEncoder(name='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>


def add(reader, writer, column, start, stop, value):

for i, row in enumerate(reader):<EOL><INDENT>if i >= start and i <= stop:<EOL><INDENT>row[column] = type(value)(row[column]) + value<EOL><DEDENT>writer.appendRecord(row)<EOL><DEDENT>

Adds a value over a range of rows.

    Args:
      reader: A FileRecordStream object with input data.
      writer: A FileRecordStream object to write output data to.
      column: The column of data to modify.
      start: The first row in the range to modify.
      end: The last row in the range to modify.
      value: The value to add.


def scale(reader, writer, column, start, stop, multiple):

for i, row in enumerate(reader):<EOL><INDENT>if i >= start and i <= stop:<EOL><INDENT>row[column] = type(multiple)(row[column]) * multiple<EOL><DEDENT>writer.appendRecord(row)<EOL><DEDENT>

Multiplies a value over a range of rows.

    Args:
      reader: A FileRecordStream object with input data.
      writer: A FileRecordStream  object to write output data to.
      column: The column of data to modify.
      start: The first row in the range to modify.
      end: The last row in the range to modify.
      multiple: The value to scale/multiply by.


def copy(reader, writer, start, stop, insertLocation=None, tsCol=None):

assert stop >= start<EOL>startRows = []<EOL>copyRows = []<EOL>ts = None<EOL>inc = None<EOL>if tsCol is None:<EOL><INDENT>tsCol = reader.getTimestampFieldIdx()<EOL><DEDENT>for i, row in enumerate(reader):<EOL><INDENT>if ts is None:<EOL><INDENT>ts = row[tsCol]<EOL><DEDENT>elif inc is None:<EOL><INDENT>inc = row[tsCol] - ts<EOL><DEDENT>if i >= start and i <= stop:<EOL><INDENT>copyRows.append(row)<EOL><DEDENT>startRows.append(row)<EOL><DEDENT>if insertLocation is None:<EOL><INDENT>insertLocation = stop + <NUM_LIT:1><EOL><DEDENT>startRows[insertLocation:insertLocation] = copyRows<EOL>for row in startRows:<EOL><INDENT>row[tsCol] = ts<EOL>writer.appendRecord(row)<EOL>ts += inc<EOL><DEDENT>

Copies a range of values to a new location in the data set.

    Args:
      reader: A FileRecordStream object with input data.
      writer: A FileRecordStream object to write output data to.
      start: The first row in the range to copy.
      stop: The last row in the range to copy.
      insertLocation: The location to insert the copied range. If not specified,
          the range is inserted immediately following itself.


def sample(reader, writer, n, start=None, stop=None, tsCol=None,<EOL>writeSampleOnly=True):

rows = list(reader)<EOL>if tsCol is not None:<EOL><INDENT>ts = rows[<NUM_LIT:0>][tsCol]<EOL>inc = rows[<NUM_LIT:1>][tsCol] - ts<EOL><DEDENT>if start is None:<EOL><INDENT>start = <NUM_LIT:0><EOL><DEDENT>if stop is None:<EOL><INDENT>stop = len(rows) - <NUM_LIT:1><EOL><DEDENT>initialN = stop - start + <NUM_LIT:1><EOL>numDeletes =  initialN - n<EOL>for i in range(numDeletes):<EOL><INDENT>delIndex = random.randint(start, stop - i)<EOL>del rows[delIndex]<EOL><DEDENT>if writeSampleOnly:<EOL><INDENT>rows = rows[start:start + n]<EOL><DEDENT>if tsCol is not None:<EOL><INDENT>ts = rows[<NUM_LIT:0>][tsCol]<EOL><DEDENT>for row in rows:<EOL><INDENT>if tsCol is not None:<EOL><INDENT>row[tsCol] = ts<EOL>ts += inc<EOL><DEDENT>writer.appendRecord(row)<EOL><DEDENT>

Samples n rows.

    Args:
      reader: A FileRecordStream object with input data.
      writer: A FileRecordStream object to write output data to.
      n: The number of elements to sample.
      start: The first row in the range to sample from.
      stop: The last row in the range to sample from.
      tsCol: If specified, the timestamp column to update.
      writeSampleOnly: If False, the rows before start are written before the
          sample and the rows after stop are written after the sample.


A distribution is a set of values with certain statistical properties

        Methods/properties that must be implemented by subclasses
        - getNext() -- Returns the next value for the distribution
        - getData(n) -- Returns n values for the distribution
        - getDescription() -- Returns a dict of parameters pertinent to the
          distribution, if any as well as state variables.


raise Exception("<STR_LIT>")<EOL>

Returns the next value of the disribution using knowledge about the
        current state of the distribution as stored in numValues.


records = [self.getNext() for x in range(n)]<EOL>return records<EOL>

Returns the next n values for the distribution as a list.


raise Exception("<STR_LIT>")<EOL>

Returns a dict of parameters pertinent to the distribution (if any) as
        well as state variables such as numValues.


hasReset = self.resetFieldName is not None<EOL>hasSequenceId = self.sequenceIdFieldName is not None<EOL>if hasReset and not hasSequenceId:<EOL><INDENT>self._sequenceInfoType = self.SEQUENCEINFO_RESET_ONLY<EOL>self._prevSequenceId = <NUM_LIT:0><EOL><DEDENT>elif not hasReset and hasSequenceId:<EOL><INDENT>self._sequenceInfoType = self.SEQUENCEINFO_SEQUENCEID_ONLY<EOL>self._prevSequenceId = None<EOL><DEDENT>elif hasReset and hasSequenceId:<EOL><INDENT>self._sequenceInfoType = self.SEQUENCEINFO_BOTH<EOL><DEDENT>else:<EOL><INDENT>self._sequenceInfoType = self.SEQUENCEINFO_NONE<EOL><DEDENT>

Figure out whether reset, sequenceId,
        both or neither are present in the data.
        Compute once instead of every time.

        Taken from filesource.py


inferencesToWrite = {}<EOL>if self._inferenceBuffer is None:<EOL><INDENT>maxDelay = InferenceElement.getMaxDelay(modelResult.inferences)<EOL>self._inferenceBuffer = collections.deque(maxlen=maxDelay + <NUM_LIT:1>)<EOL><DEDENT>self._inferenceBuffer.appendleft(copy.deepcopy(modelResult.inferences))<EOL>for inferenceElement, inference in modelResult.inferences.iteritems():<EOL><INDENT>if isinstance(inference, dict):<EOL><INDENT>inferencesToWrite[inferenceElement] = {}<EOL>for key, _ in inference.iteritems():<EOL><INDENT>delay = InferenceElement.getTemporalDelay(inferenceElement, key)<EOL>if len(self._inferenceBuffer) > delay:<EOL><INDENT>prevInference = self._inferenceBuffer[delay][inferenceElement][key]<EOL>inferencesToWrite[inferenceElement][key] = prevInference<EOL><DEDENT>else:<EOL><INDENT>inferencesToWrite[inferenceElement][key] = None<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>delay = InferenceElement.getTemporalDelay(inferenceElement)<EOL>if len(self._inferenceBuffer) > delay:<EOL><INDENT>inferencesToWrite[inferenceElement] = (<EOL>self._inferenceBuffer[delay][inferenceElement])<EOL><DEDENT>else:<EOL><INDENT>if type(inference) in (list, tuple):<EOL><INDENT>inferencesToWrite[inferenceElement] = [None] * len(inference)<EOL><DEDENT>else:<EOL><INDENT>inferencesToWrite[inferenceElement] = None<EOL><DEDENT><DEDENT><DEDENT><DEDENT>shiftedResult = ModelResult(rawInput=modelResult.rawInput,<EOL>sensorInput=modelResult.sensorInput,<EOL>inferences=inferencesToWrite,<EOL>metrics=modelResult.metrics,<EOL>predictedFieldIdx=modelResult.predictedFieldIdx,<EOL>predictedFieldName=modelResult.predictedFieldName)<EOL>return shiftedResult<EOL>

Shift the model result and return the new instance.

        Queues up the T(i+1) prediction value and emits a T(i)
        input/prediction pair, if possible. E.g., if the previous T(i-1)
        iteration was learn-only, then we would not have a T(i) prediction in our
        FIFO and would not be able to emit a meaningful input/prediction pair.

        :param modelResult: A :class:`~.nupic.frameworks.opf.opf_utils.ModelResult`
                            instance to shift.
        :return: A :class:`~.nupic.frameworks.opf.opf_utils.ModelResult` instance that
                 has been shifted


assert len(list(kwds.keys())) >= <NUM_LIT:1><EOL>assert '<STR_LIT>' in kwds or '<STR_LIT>' in kwds<EOL>schemaDict = None<EOL>if '<STR_LIT>' in kwds:<EOL><INDENT>schemaPath = kwds.pop('<STR_LIT>')<EOL>schemaDict = loadJsonValueFromFile(schemaPath)<EOL><DEDENT>elif '<STR_LIT>' in kwds:<EOL><INDENT>schemaDict = kwds.pop('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>validictory.validate(value, schemaDict, **kwds)<EOL><DEDENT>except validictory.ValidationError as e:<EOL><INDENT>raise ValidationError(e)<EOL><DEDENT>

Validate a python value against json schema:
    validate(value, schemaPath)
    validate(value, schemaDict)

    value:          python object to validate against the schema

    The json schema may be specified either as a path of the file containing
    the json schema or as a python dictionary using one of the
    following keywords as arguments:
      schemaPath:     Path of file containing the json schema object.
      schemaDict:     Python dictionary containing the json schema object

    Returns: nothing

    Raises:
            ValidationError when value fails json validation


with open(inputFilePath) as fileObj:<EOL><INDENT>value = json.load(fileObj)<EOL><DEDENT>return value<EOL>

Loads a json value from a file and converts it to the corresponding python
    object.

    inputFilePath:
                    Path of the json file;

    Returns:
                    python value that represents the loaded json value


pass<EOL>

Get Cap'n Proto schema.

..warning: This is an abstract method.  Per abc protocol, attempts to subclass
           without overriding will fail.

@returns Cap'n Proto schema


pass<EOL>

Create a new object initialized from Cap'n Proto obj.

Note: This is an abstract method.  Per abc protocol, attempts to subclass
without overriding will fail.

:param proto: Cap'n Proto obj
:return: Obj initialized from proto


pass<EOL>

Write obj instance to Cap'n Proto object

.. warning: This is an abstract method.  Per abc protocol, attempts to
            subclass without overriding will fail.

:param proto: Cap'n Proto obj


@classmethod<EOL><INDENT>def readFromFile(cls, f, packed=True):<DEDENT>

<EOL>schema = cls.getSchema()<EOL>if packed:<EOL><INDENT>proto = schema.read_packed(f)<EOL><DEDENT>else:<EOL><INDENT>proto = schema.read(f)<EOL><DEDENT>return cls.read(proto)<EOL>

Read serialized object from file.

:param f: input file
:param packed: If true, will assume content is packed
:return: first-class instance initialized from proto obj


<EOL>schema = self.getSchema()<EOL>proto = schema.new_message()<EOL>self.write(proto)<EOL>if packed:<EOL><INDENT>proto.write_packed(f)<EOL><DEDENT>else:<EOL><INDENT>proto.write(f)<EOL><DEDENT>

Write serialized object to file.

:param f: output file
:param packed: If true, will pack contents.


graph = nx.MultiDiGraph()<EOL>regions = self.network.getRegions()<EOL>for idx in xrange(regions.getCount()):<EOL><INDENT>regionPair = regions.getByIndex(idx)<EOL>regionName = regionPair[<NUM_LIT:0>]<EOL>graph.add_node(regionName, label=regionName)<EOL><DEDENT>for linkName, link in self.network.getLinks():<EOL><INDENT>graph.add_edge(link.getSrcRegionName(),<EOL>link.getDestRegionName(),<EOL>src=link.getSrcOutputName(),<EOL>dest=link.getDestInputName())<EOL><DEDENT>return graph<EOL>

Exports a network as a networkx MultiDiGraph intermediate representation
suitable for visualization.

:return: networkx MultiDiGraph


renderer().render(self.export())<EOL>

Render network. Default is
:class:`~nupic.frameworks.viz.dot_renderer.DotRenderer`.

:param renderer: Constructor parameter to a "renderer" implementation.
       Return value for which must have a "render" method that accepts a
       single argument (a networkx graph instance).


self.supported_nodes = tuple(set(self.supported_nodes) -<EOL>self.blacklisted_nodes)<EOL>asteval.Interpreter.__init__(self, *args, **kwargs)<EOL>

Initialize interpreter with blacklisted nodes removed from supported
        nodes.


metricName = metricSpec.metric<EOL>if metricName == '<STR_LIT>':<EOL><INDENT>return MetricRMSE(metricSpec)<EOL><DEDENT>if metricName == '<STR_LIT>':<EOL><INDENT>return MetricNRMSE(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricAAE(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricAccuracy(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricAveError(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricTrivial(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricTwoGram(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMovingMean(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMovingMode(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricNegAUC(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return CustomErrorMetric(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMultiStep(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMultiStepProbability(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMultiStepAAE(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMultiStepAveError(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricPassThruPrediction(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricAltMAPE(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMAPE(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricMulti(metricSpec)<EOL><DEDENT>elif metricName == '<STR_LIT>':<EOL><INDENT>return MetricNegativeLogLikelihood(metricSpec)<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % metricName)<EOL><DEDENT>

Factory method to return an appropriate :class:`MetricsIface` module.

- ``rmse``: :class:`MetricRMSE`
- ``nrmse``: :class:`MetricNRMSE`
- ``aae``: :class:`MetricAAE`
- ``acc``: :class:`MetricAccuracy`
- ``avg_err``: :class:`MetricAveError`
- ``trivial``: :class:`MetricTrivial`
- ``two_gram``: :class:`MetricTwoGram`
- ``moving_mean``: :class:`MetricMovingMean`
- ``moving_mode``: :class:`MetricMovingMode`
- ``neg_auc``: :class:`MetricNegAUC`
- ``custom_error_metric``: :class:`CustomErrorMetric`
- ``multiStep``: :class:`MetricMultiStep`
- ``ms_aae``: :class:`MetricMultiStepAAE`
- ``ms_avg_err``: :class:`MetricMultiStepAveError`
- ``passThruPrediction``: :class:`MetricPassThruPrediction`
- ``altMAPE``: :class:`MetricAltMAPE`
- ``MAPE``: :class:`MetricMAPE`
- ``multi``: :class:`MetricMulti`
- ``negativeLogLikelihood``: :class:`MetricNegativeLogLikelihood`

:param metricSpec: (:class:`MetricSpec`) metric to find module for.
       ``metricSpec.metric`` must be in the list above.

:returns: (:class:`AggregateMetric`) an appropriate metric module


result = []<EOL>if inferenceType is not None:<EOL><INDENT>result.append(InferenceType.getLabel(inferenceType))<EOL><DEDENT>result.append(self.inferenceElement)<EOL>result.append(self.metric)<EOL>params = self.params<EOL>if params is not None:<EOL><INDENT>sortedParams= list(params.keys())<EOL>sortedParams.sort()<EOL>for param in sortedParams:<EOL><INDENT>if param in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>value = params[param]<EOL>if isinstance(value, str):<EOL><INDENT>result.extend(["<STR_LIT>"% (param, value)])<EOL><DEDENT>else:<EOL><INDENT>result.extend(["<STR_LIT>"% (param, value)])<EOL><DEDENT><DEDENT><DEDENT>if self.field:<EOL><INDENT>result.append("<STR_LIT>"% (self.field) )<EOL><DEDENT>return self._LABEL_SEPARATOR.join(result)<EOL>

Helper method that generates a unique label for a :class:`MetricSpec` /
:class:`~nupic.frameworks.opf.opf_utils.InferenceType` pair. The label is
formatted as follows:

::

    <predictionKind>:<metric type>:(paramName=value)*:field=<fieldname>

For example:

::

    classification:aae:paramA=10.2:paramB=20:window=100:field=pounds

:returns: (string) label for inference type


infType, _, _= label.partition(cls._LABEL_SEPARATOR)<EOL>if not InferenceType.validate(infType):<EOL><INDENT>return None<EOL><DEDENT>return infType<EOL>

Extracts the PredictionKind (temporal vs. nontemporal) from the given
metric label.

:param label: (string) for a metric spec generated by
       :meth:`getMetricLabel`

:returns: (:class:`~nupic.frameworks.opf.opf_utils.InferenceType`)


self._windowSize = windowSize<EOL>self._countDict = dict()<EOL>self._history = deque([])<EOL>

:param windowSize:             The number of values that are used to compute the
                        moving average


@abstractmethod<EOL><INDENT>def addInstance(self, groundTruth, prediction, record = None, result = None):<DEDENT>

Add one instance consisting of ground truth and a prediction.

:param groundTruth:
  The actual measured value at the current timestep

:param prediction:
  The value predicted by the network at the current timestep

:param record: the raw input record as fed to
       :meth:`~nupic.frameworks.opf.model.Model.run` by the user. The
       typical usage is to feed a record to that method and get a
       :class:`~nupic.frameworks.opf.opf_utils.ModelResult`. Then you pass
       :class:`~nupic.frameworks.opf.opf_utils.ModelResult`.rawInput into
       this function as the record parameter.

:param result: (:class:`~nupic.frameworks.opf.opf_utils.ModelResult`) the
       result of running a row of data through an OPF model

:returns:
    The average error as computed over the metric's window size


``stats`` is expected to contain further information relevant to the given
metric, for example the number of timesteps represented in the current
measurement. All stats are implementation defined, and ``stats`` can be
``None``.

:returns: (dict) representing data from the metric
   ::

       {value : <current measurement>, "stats" : {<stat> : <value> ...}}


def accumulate(self, groundTruth, prediction, accumulatedError, historyBuffer, result):

Updates the accumulated error given the prediction and the
ground truth.

:param groundTruth: Actual value that is observed for the current timestep

:param prediction: Value predicted by the network for the given timestep

:param accumulatedError: The total accumulated score from the previous
       predictions (possibly over some finite window)

:param historyBuffer: A buffer of the last <self.window> ground truth values
       that have been observed.

       If historyBuffer = None,  it means that no history is being kept.

:param result: An ModelResult class (see opf_utils.py), used for advanced
       metric calculation (e.g., MetricNegativeLogLikelihood)

:returns: The new accumulated error. That is:

    .. code-block:: python

       self.accumulatedError = self.accumulate(
         groundTruth, predictions, accumulatedError
       )

    ``historyBuffer`` should also be updated in this method.
    ``self.spec.params["window"]`` indicates the maximum size of the window.


def aggregate(self, accumulatedError, historyBuffer, steps):

Updates the final aggregated score error given the prediction and the ground
truth.

:param accumulatedError: The total accumulated score from the previous
       predictions (possibly over some finite window)

:param historyBuffer: A buffer of the last <self.window> ground truth values
       that have been observed. If ``historyBuffer`` = None,  it means that
       no history is being kept.

:param steps: (int) The total number of (groundTruth, prediction) pairs that
       have been passed to the metric. This does not include pairs where
       ``groundTruth = SENTINEL_VALUE_FOR_MISSING_DATA``

:returns: The new aggregate (final) error measure.


<EOL>self.id = None<EOL>self.verbosity = <NUM_LIT:0><EOL>self.window = -<NUM_LIT:1><EOL>self.history = None<EOL>self.accumulatedError = <NUM_LIT:0><EOL>self.aggregateError = None<EOL>self.steps = <NUM_LIT:0><EOL>self.spec = metricSpec<EOL>self.disabled = False<EOL>self._predictionSteps = [<NUM_LIT:0>]<EOL>self._groundTruthHistory = deque([])<EOL>self._subErrorMetrics = None<EOL>self._maxRecords = None<EOL>if metricSpec is not None and metricSpec.params is not None:<EOL><INDENT>self.id = metricSpec.params.get('<STR_LIT:id>', None)<EOL>self._predictionSteps = metricSpec.params.get('<STR_LIT>', [<NUM_LIT:0>])<EOL>if not hasattr(self._predictionSteps, '<STR_LIT>'):<EOL><INDENT>self._predictionSteps = [self._predictionSteps]<EOL><DEDENT>self.verbosity = metricSpec.params.get('<STR_LIT>', <NUM_LIT:0>)<EOL>self._maxRecords = metricSpec.params.get('<STR_LIT>', None)<EOL>if '<STR_LIT>' in metricSpec.params:<EOL><INDENT>assert metricSpec.params['<STR_LIT>'] >= <NUM_LIT:1><EOL>self.history = deque([])<EOL>self.window = metricSpec.params['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in metricSpec.params:<EOL><INDENT>self._subErrorMetrics = []<EOL>for step in self._predictionSteps:<EOL><INDENT>subSpec = copy.deepcopy(metricSpec)<EOL>subSpec.params.pop('<STR_LIT>', None)<EOL>subSpec.params.pop('<STR_LIT>')<EOL>subSpec.metric = metricSpec.params['<STR_LIT>']<EOL>self._subErrorMetrics.append(getModule(subSpec))<EOL><DEDENT><DEDENT><DEDENT>

Initialize this metric

        If the params contains the key 'errorMetric', then that is the name of
        another metric to which we will pass a modified groundTruth and prediction
        to from our addInstance() method. For example, we may compute a moving mean
        on the groundTruth and then pass that to the AbsoluteAveError metric


<EOL>self._groundTruthHistory.append(groundTruth)<EOL>assert (len(self._predictionSteps) == <NUM_LIT:1>)<EOL>if len(self._groundTruthHistory) > self._predictionSteps[<NUM_LIT:0>]:<EOL><INDENT>return self._groundTruthHistory.popleft()<EOL><DEDENT>else:<EOL><INDENT>if hasattr(groundTruth, '<STR_LIT>'):<EOL><INDENT>return [None] * len(groundTruth)<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>

Utility function that saves the passed in groundTruth into a local
        history buffer, and returns the groundTruth from self._predictionSteps ago,
        where self._predictionSteps is defined by the 'steps' parameter.
        This can be called from the beginning of a derived class's addInstance()
        before it passes groundTruth and prediction onto accumulate().


def addInstance(self, groundTruth, prediction, record = None, result = None):

self.value = self.avg(prediction)<EOL>


return {"<STR_LIT:value>": self.value}<EOL>


if len(pred) == <NUM_LIT:1>:<EOL><INDENT>return list(pred.keys())[<NUM_LIT:0>]<EOL><DEDENT>mostLikelyOutcome = None<EOL>maxProbability = <NUM_LIT:0><EOL>for prediction, probability in list(pred.items()):<EOL><INDENT>if probability > maxProbability:<EOL><INDENT>mostLikelyOutcome = prediction<EOL>maxProbability = probability<EOL><DEDENT><DEDENT>return mostLikelyOutcome<EOL>

Helper function to return a scalar value representing the most
            likely outcome given a probability distribution


if len(pred) == <NUM_LIT:1>:<EOL><INDENT>return list(pred.keys())[<NUM_LIT:0>]<EOL><DEDENT>return sum([x*p for x,p in list(pred.items())])<EOL>

Helper function to return a scalar value representing the expected
            value of a probability distribution


def accumulate(self, groundTruth, prediction, accumulatedError, historyBuffer, result = None):

<EOL>if self.disabled:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>if historyBuffer is not None:<EOL><INDENT>historyBuffer.append((groundTruth, prediction[<NUM_LIT:0>]))<EOL>if len(historyBuffer) > self.spec.params["<STR_LIT>"] :<EOL><INDENT>historyBuffer.popleft()<EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>

Accumulate history of groundTruth and "prediction" values.

For this metric, groundTruth is the actual category and "prediction" is a
dict containing one top-level item with a key of 0 (meaning this is the
0-step classificaton) and a value which is another dict, which contains the
probability for each category as output from the classifier. For example,
this is what "prediction" would be if the classifier said that category 0
had a 0.6 probability and category 1 had a 0.4 probability: {0:0.6, 1: 0.4}


raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>

MetricMulti constructor using metricSpec is not allowed.


def __init__(self, weights, metrics, window=None):

if (weights is None or not isinstance(weights, list) or<EOL>not len(weights) > <NUM_LIT:0> or<EOL>not isinstance(weights[<NUM_LIT:0>], float)):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.weights = weights<EOL>if (metrics is None or not isinstance(metrics, list) or<EOL>not len(metrics) > <NUM_LIT:0> or<EOL>not isinstance(metrics[<NUM_LIT:0>], MetricsIface)):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.metrics = metrics<EOL>if window is not None:<EOL><INDENT>self.movingAvg = MovingAverage(windowSize=window)<EOL><DEDENT>else:<EOL><INDENT>self.movingAvg = None<EOL><DEDENT>

MetricMulti
           @param weights - [list of floats] used as weights
           @param metrics - [list of submetrics]
           @param window - (opt) window size for moving average, or None when disabled


assert isinstance(htmPredictionModel, HTMPredictionModel)<EOL>htmPredictionModel._getSPRegion().setParameter('<STR_LIT>', True)<EOL>return<EOL>

Enables learning in the HTMPredictionModel's Spatial Pooler

    See also htmPredictionModelControlDisableSPLearningCb.

    htmPredictionModel:  pointer to a HTMPredictionModel instance

    Returns: nothing


assert isinstance(htmPredictionModel, HTMPredictionModel)<EOL>htmPredictionModel._getSPRegion().setParameter('<STR_LIT>', False)<EOL>return<EOL>

Disables learning in the HTMPredictionModel's Spatial Pooler, while
    retaining the ability to re-enable SP learning in the future.

    See also: htmPredictionModelControlEnableSPLearningCb.
    See also: model_callbacks.modelControlFinishLearningCb.

    htmPredictionModel:  pointer to a HTMPredictionModel instance

    Returns: nothing


assert isinstance(htmPredictionModel, HTMPredictionModel)<EOL>htmPredictionModel._getTPRegion().setParameter('<STR_LIT>', True)<EOL>return<EOL>

Enables learning in the HTMPredictionModel's Temporal Pooler

    See also htmPredictionModelControlDisableTPLearningCb.

    htmPredictionModel:  pointer to a HTMPredictionModel instance

    Returns: nothing


assert isinstance(htmPredictionModel, HTMPredictionModel)<EOL>htmPredictionModel._getTPRegion().setParameter('<STR_LIT>', False)<EOL>return<EOL>

Disables learning in the HTMPredictionModel's Temporal Pooler, while
    retaining the ability to re-enable TM learning in the future.

    See also: htmPredictionModelControlEnableTPLearningCb.
    See also: model_callbacks.modelControlFinishLearningCb.

    htmPredictionModel:  pointer to a HTMPredictionModel instance

    Returns: nothing


self.__filePath = filePath<EOL>return<EOL>

filePath: path of file where SP __init__ args are to be saved


filePath: path of file where TM __init__ args are to be saved


<EOL>opt = _parseCommandLineOptions(args)<EOL>model = _runExperimentImpl(opt, model)<EOL>return model<EOL>

Run a single OPF experiment.

.. note:: The caller is responsible for initializing python logging before
   calling this function (e.g., import :mod:`nupic.support`;
   :meth:`nupic.support.initLogging`)

See also: :meth:`.initExperimentPrng`.

:param args: (string) Experiment command-line args list. Too see all options,
    run with ``--help``:

    .. code-block:: text

      Options:
        -h, --help           show this help message and exit
        -c <CHECKPOINT>      Create a model and save it under the given <CHECKPOINT>
                             name, but don't run it
        --listCheckpoints    List all available checkpoints
        --listTasks          List all task labels in description.py
        --load=<CHECKPOINT>  Load a model from the given <CHECKPOINT> and run it.
                             Run with --listCheckpoints flag for more details.
        --newSerialization   Use new capnproto serialization
        --tasks              Run the tasks with the given TASK LABELS in the order
                             they are given.  Either end of arg-list, or a
                             standalone dot ('.') arg or the next short or long
                             option name (-a or --blah) terminates the list. NOTE:
                             FAILS TO RECOGNIZE task label names with one or more
                             leading dashes. [default: run all of the tasks in
                             description.py]
        --testMode           Reduce iteration count for testing
        --noCheckpoint       Don't checkpoint the model after running each task.

:param model: (:class:`~nupic.frameworks.opf.model.Model`) For testing, may
    pass in an existing OPF Model to use instead of creating a new one.

:returns: (:class:`~nupic.frameworks.opf.model.Model`)
  reference to OPF Model instance that was constructed (this
  is provided to aid with debugging) or None, if none was
  created.


seed = <NUM_LIT><EOL>random.seed(seed)<EOL>numpy.random.seed(seed)<EOL>

Initialize PRNGs that may be used by other modules in the experiment stack.

    .. note:: User may call this function to initialize PRNGs that are used by the
       experiment stack before calling runExperiment(), unless user has its own
       own logic for initializing these PRNGs.


Parse command line options

    Args:
      args: command line arguments (not including sys.argv[0])
    Returns:
      namedtuple ParseCommandLineOptionsResult


def reapVarArgsCallback(option, optStr, value, parser):

newValues = []<EOL>gotDot = False<EOL>for arg in parser.rargs:<EOL><INDENT>if arg.startswith("<STR_LIT>") and len(arg) > <NUM_LIT:2>:<EOL><INDENT>break<EOL><DEDENT>if arg.startswith("<STR_LIT:->") and len(arg) > <NUM_LIT:1>:<EOL><INDENT>break<EOL><DEDENT>if arg == "<STR_LIT:.>":<EOL><INDENT>gotDot = True<EOL>break<EOL><DEDENT>newValues.append(arg)<EOL><DEDENT>if not newValues:<EOL><INDENT>raise optparse.OptionValueError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>") % (optStr, parser.rargs))<EOL><DEDENT>del parser.rargs[:len(newValues) + int(gotDot)]<EOL>value = getattr(parser.values, option.dest, [])<EOL>if value is None:<EOL><INDENT>value = []<EOL><DEDENT>value.extend(newValues)<EOL>setattr(parser.values, option.dest, value)<EOL>

Used as optparse callback for reaping a variable number of option args.
    The option may be specified multiple times, and all the args associated with
    that option name will be accumulated in the order that they are encountered


print(parser.get_usage())<EOL>print(message)<EOL>sys.exit(<NUM_LIT:1>)<EOL>

Report usage error and exit program with error indication.


json_helpers.validate(options.privateOptions,<EOL>schemaDict=g_parsedPrivateCommandLineOptionsSchema)<EOL>experimentDir = options.experimentDir<EOL>descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(<EOL>experimentDir)<EOL>expIface = helpers.getExperimentDescriptionInterfaceFromModule(<EOL>descriptionPyModule)<EOL>if options.privateOptions['<STR_LIT>']:<EOL><INDENT>_printAvailableCheckpoints(experimentDir)<EOL>return None<EOL><DEDENT>experimentTasks = expIface.getModelControl().get('<STR_LIT>', [])<EOL>if (len(experimentTasks) == <NUM_LIT:0> and<EOL>expIface.getModelControl()['<STR_LIT>'] == OpfEnvironment.Nupic):<EOL><INDENT>expIface.convertNupicEnvToOPF()<EOL>experimentTasks = expIface.getModelControl().get('<STR_LIT>', [])<EOL><DEDENT>expIface.normalizeStreamSources()<EOL>newSerialization = options.privateOptions['<STR_LIT>']<EOL>if options.privateOptions['<STR_LIT>']:<EOL><INDENT>print("<STR_LIT>")<EOL>for label in [t['<STR_LIT>'] for t in experimentTasks]:<EOL><INDENT>print("<STR_LIT:\t>", label)<EOL><DEDENT>return None<EOL><DEDENT>if options.privateOptions['<STR_LIT>']:<EOL><INDENT>assert model is None<EOL>checkpointName = options.privateOptions['<STR_LIT>']<EOL>model = ModelFactory.loadFromCheckpoint(<EOL>savedModelDir=_getModelCheckpointDir(experimentDir, checkpointName),<EOL>newSerialization=newSerialization)<EOL><DEDENT>elif model is not None:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>modelDescription = expIface.getModelDescription()<EOL>model = ModelFactory.create(modelDescription)<EOL><DEDENT>if options.privateOptions['<STR_LIT>']:<EOL><INDENT>checkpointName = options.privateOptions['<STR_LIT>']<EOL>_saveModel(model=model,<EOL>experimentDir=experimentDir,<EOL>checkpointLabel=checkpointName,<EOL>newSerialization=newSerialization)<EOL>return model<EOL><DEDENT>taskIndexList = list(range(len(experimentTasks)))<EOL>customTaskExecutionLabelsList = options.privateOptions['<STR_LIT>']<EOL>if customTaskExecutionLabelsList:<EOL><INDENT>taskLabelsList = [t['<STR_LIT>'] for t in experimentTasks]<EOL>taskLabelsSet = set(taskLabelsList)<EOL>customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)<EOL>assert customTaskExecutionLabelsSet.issubset(taskLabelsSet),("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % (customTaskExecutionLabelsSet - taskLabelsSet,<EOL>customTaskExecutionLabelsList)<EOL>taskIndexList = [taskLabelsList.index(label) for label in<EOL>customTaskExecutionLabelsList]<EOL>print("<STR_LIT>" % [taskLabelsList[i] for<EOL>i in taskIndexList])<EOL><DEDENT>for taskIndex in taskIndexList:<EOL><INDENT>task = experimentTasks[taskIndex]<EOL>taskRunner = _TaskRunner(model=model,<EOL>task=task,<EOL>cmdOptions=options)<EOL>taskRunner.run()<EOL>del taskRunner<EOL>if options.privateOptions['<STR_LIT>']:<EOL><INDENT>_saveModel(model=model,<EOL>experimentDir=experimentDir,<EOL>checkpointLabel=task['<STR_LIT>'],<EOL>newSerialization=newSerialization)<EOL><DEDENT><DEDENT>return model<EOL>

Creates and runs the experiment

    Args:
      options: namedtuple ParseCommandLineOptionsResult
      model: For testing: may pass in an existing OPF Model instance
          to use instead of creating a new one.

    Returns: reference to OPFExperiment instance that was constructed (this
        is provided to aid with debugging) or None, if none was
        created.


def _saveModel(model, experimentDir, checkpointLabel, newSerialization=False):

checkpointDir = _getModelCheckpointDir(experimentDir, checkpointLabel)<EOL>if newSerialization:<EOL><INDENT>model.writeToCheckpoint(checkpointDir)<EOL><DEDENT>else:<EOL><INDENT>model.save(saveModelDir=checkpointDir)<EOL><DEDENT>


checkpointDir = os.path.join(getCheckpointParentDir(experimentDir),<EOL>checkpointLabel + g_defaultCheckpointExtension)<EOL>checkpointDir = os.path.abspath(checkpointDir)<EOL>return checkpointDir<EOL>

Creates directory for serialization of the model

    checkpointLabel:
        Checkpoint label (string)

    Returns:
      absolute path to the serialization directory


baseDir = os.path.join(experimentDir, "<STR_LIT>")<EOL>baseDir = os.path.abspath(baseDir)<EOL>return baseDir<EOL>

Get checkpoint parent dir.

    Returns: absolute path to the base serialization directory within which
        model checkpoints for this experiment are created


assert checkpointDir.endswith(g_defaultCheckpointExtension)<EOL>lastSegment = os.path.split(checkpointDir)[<NUM_LIT:1>]<EOL>checkpointLabel = lastSegment[<NUM_LIT:0>:-len(g_defaultCheckpointExtension)]<EOL>return checkpointLabel<EOL>

Returns a checkpoint label string for the given model checkpoint directory

    checkpointDir: relative or absolute model checkpoint directory path


lastSegment = os.path.split(checkpointDir)[<NUM_LIT:1>]<EOL>if lastSegment[<NUM_LIT:0>] == '<STR_LIT:.>':<EOL><INDENT>return False<EOL><DEDENT>if not checkpointDir.endswith(g_defaultCheckpointExtension):<EOL><INDENT>return False<EOL><DEDENT>if not os.path.isdir(checkpointDir):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Return true iff checkpointDir appears to be a checkpoint directory.


checkpointParentDir = getCheckpointParentDir(experimentDir)<EOL>if not os.path.exists(checkpointParentDir):<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>checkpointDirs = [x for x in os.listdir(checkpointParentDir)<EOL>if _isCheckpointDir(os.path.join(checkpointParentDir, x))]<EOL>if not checkpointDirs:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>print("<STR_LIT>")<EOL>checkpointList = [_checkpointLabelFromCheckpointDir(x)<EOL>for x in checkpointDirs]<EOL>for checkpoint in sorted(checkpointList):<EOL><INDENT>print("<STR_LIT:\t>", checkpoint)<EOL><DEDENT>print()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>

List available checkpoints for the specified experiment.


initLogging(verbose=True)<EOL>initExperimentPrng()<EOL>runExperiment(sys.argv[<NUM_LIT:1>:])<EOL>

Module-level entry point.  Run according to options in sys.argv

    Usage: python -m python -m nupic.frameworks.opf.experiment_runner


def __init__(self, model, task, cmdOptions):

validateOpfJsonValue(task, "<STR_LIT>")<EOL>self.__logger = logging.getLogger("<STR_LIT:.>".join(<EOL>['<STR_LIT>', self.__class__.__module__, self.__class__.__name__]))<EOL>self.__logger.debug(("<STR_LIT>" +"<STR_LIT>" +"<STR_LIT>" +"<STR_LIT>") %(self.__class__.__name__,<EOL>model,<EOL>task,<EOL>cmdOptions))<EOL>streamDef = task['<STR_LIT>']<EOL>datasetReader = opf_basic_environment.BasicDatasetReader(streamDef)<EOL>self.__model = model<EOL>self.__datasetReader = datasetReader<EOL>self.__task = task<EOL>self.__cmdOptions = cmdOptions<EOL>self.__predictionLogger = opf_basic_environment.BasicPredictionLogger(<EOL>fields=model.getFieldInfo(),<EOL>experimentDir=cmdOptions.experimentDir,<EOL>label=task['<STR_LIT>'],<EOL>inferenceType=self.__model.getInferenceType())<EOL>taskControl = task['<STR_LIT>']<EOL>self.__taskDriver = OPFTaskDriver(<EOL>taskControl=taskControl,<EOL>model=model)<EOL>loggedMetricPatterns = taskControl.get('<STR_LIT>', None)<EOL>loggedMetricLabels = matchPatterns(loggedMetricPatterns,<EOL>self.__taskDriver.getMetricLabels())<EOL>self.__predictionLogger.setLoggedMetrics(loggedMetricLabels)<EOL>self.__metricsLogger = opf_basic_environment.BasicPredictionMetricsLogger(<EOL>experimentDir=cmdOptions.experimentDir,<EOL>label=task['<STR_LIT>'])<EOL>

Constructor

        Args:
          model: The OPF Model instance against which to run the task
          task: A dictionary conforming to opfTaskSchema.json
          cmdOptions: ParseCommandLineOptionsResult namedtuple


<EOL>


self.__logger.debug("<STR_LIT>", self.__task['<STR_LIT>'])<EOL>if self.__cmdOptions.privateOptions['<STR_LIT>']:<EOL><INDENT>numIters = <NUM_LIT:10><EOL><DEDENT>else:<EOL><INDENT>numIters = self.__task['<STR_LIT>']<EOL><DEDENT>if numIters >= <NUM_LIT:0>:<EOL><INDENT>iterTracker = iter(range(numIters))<EOL><DEDENT>else:<EOL><INDENT>iterTracker = iter(itertools.count())<EOL><DEDENT>periodic = PeriodicActivityMgr(<EOL>requestedActivities=self._createPeriodicActivities())<EOL>self.__model.resetSequenceStates()<EOL>self.__taskDriver.setup()<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>next(iterTracker)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>try:<EOL><INDENT>inputRecord = next(self.__datasetReader)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>result = self.__taskDriver.handleInputRecord(inputRecord=inputRecord)<EOL>if InferenceElement.encodings in result.inferences:<EOL><INDENT>result.inferences.pop(InferenceElement.encodings)<EOL><DEDENT>self.__predictionLogger.writeRecord(result)<EOL>periodic.tick()<EOL><DEDENT>self._getAndEmitExperimentMetrics(final=True)<EOL>self.__taskDriver.finalize()<EOL>self.__model.resetSequenceStates()<EOL>


<EOL>periodicActivities = []<EOL>class MetricsReportCb(object):<EOL><INDENT>def __init__(self, taskRunner):<EOL><INDENT>self.__taskRunner = taskRunner<EOL>return<EOL><DEDENT>def __call__(self):<EOL><INDENT>self.__taskRunner._getAndEmitExperimentMetrics()<EOL><DEDENT><DEDENT>reportMetrics = PeriodicActivityRequest(<EOL>repeating=True,<EOL>period=<NUM_LIT:1000>,<EOL>cb=MetricsReportCb(self))<EOL>periodicActivities.append(reportMetrics)<EOL>class IterationProgressCb(object):<EOL><INDENT>PROGRESS_UPDATE_PERIOD_TICKS = <NUM_LIT:1000><EOL>def __init__(self, taskLabel, requestedIterationCount, logger):<EOL><INDENT>self.__taskLabel = taskLabel<EOL>self.__requestedIterationCount = requestedIterationCount<EOL>self.__logger = logger<EOL>self.__numIterationsSoFar = <NUM_LIT:0><EOL><DEDENT>def __call__(self):<EOL><INDENT>self.__numIterationsSoFar += self.PROGRESS_UPDATE_PERIOD_TICKS<EOL>self.__logger.debug("<STR_LIT>" % (<EOL>self.__taskLabel,<EOL>self.__numIterationsSoFar,<EOL>self.__requestedIterationCount))<EOL><DEDENT><DEDENT>iterationProgressCb = IterationProgressCb(<EOL>taskLabel=self.__task['<STR_LIT>'],<EOL>requestedIterationCount=self.__task['<STR_LIT>'],<EOL>logger=self.__logger)<EOL>iterationProgressReporter = PeriodicActivityRequest(<EOL>repeating=True,<EOL>period=IterationProgressCb.PROGRESS_UPDATE_PERIOD_TICKS,<EOL>cb=iterationProgressCb)<EOL>periodicActivities.append(iterationProgressReporter)<EOL>return periodicActivities<EOL>

Creates and returns a list of activites for this TaskRunner instance

        Returns: a list of PeriodicActivityRequest elements


self.__activities = []<EOL>for req in requestedActivities:<EOL><INDENT>act =   self.Activity(repeating=req.repeating,<EOL>period=req.period,<EOL>cb=req.cb,<EOL>iteratorHolder=[iter(range(req.period-<NUM_LIT:1>))])<EOL>self.__activities.append(act)<EOL><DEDENT>

requestedActivities: a sequence of PeriodicActivityRequest elements


<EOL>for act in self.__activities:<EOL><INDENT>if not act.iteratorHolder[<NUM_LIT:0>]:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>next(act.iteratorHolder[<NUM_LIT:0>])<EOL><DEDENT>except StopIteration:<EOL><INDENT>act.cb()<EOL>if act.repeating:<EOL><INDENT>act.iteratorHolder[<NUM_LIT:0>] = iter(range(act.period-<NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>act.iteratorHolder[<NUM_LIT:0>] = None<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Activity tick handler; services all activities

        Returns:
          True if controlling iterator says it's okay to keep going;
          False to stop


def __init__(self, inferenceType=None, proto=None):

assert inferenceType is not None and proto is None or (<EOL>inferenceType is None and proto is not None), (<EOL>"<STR_LIT>")<EOL>if proto is None:<EOL><INDENT>self._numPredictions = <NUM_LIT:0><EOL>self.__inferenceType =  inferenceType<EOL>self.__learningEnabled = True<EOL>self.__inferenceEnabled = True<EOL>self.__inferenceArgs = {}<EOL><DEDENT>else:<EOL><INDENT>self._numPredictions = proto.numPredictions<EOL>inferenceType = str(proto.inferenceType)<EOL>inferenceType = inferenceType[:<NUM_LIT:1>].upper() + inferenceType[<NUM_LIT:1>:]<EOL>self.__inferenceType = InferenceType.getValue(inferenceType)<EOL>self.__learningEnabled = proto.learningEnabled<EOL>self.__inferenceEnabled = proto.inferenceEnabled<EOL>self.__inferenceArgs = json.loads(proto.inferenceArgs)<EOL><DEDENT>

:param opf_utils.InferenceType inferenceType: mutually-exclusive with proto
                                              arg
:param proto: capnp ModelProto message reader for deserializing;
              mutually-exclusive with the other constructor args.


<EOL>predictionNumber = self._numPredictions<EOL>self._numPredictions += <NUM_LIT:1><EOL>result = opf_utils.ModelResult(predictionNumber=predictionNumber,<EOL>rawInput=inputRecord)<EOL>return result<EOL>

Run one iteration of this model.

:param inputRecord: (object)
       A record object formatted according to
       :meth:`~nupic.data.record_stream.RecordStreamIface.getNextRecord` or
       :meth:`~nupic.data.record_stream.RecordStreamIface.getNextRecordDict`
       result format.
:returns: (:class:`~nupic.frameworks.opf.opf_utils.ModelResult`)
         An ModelResult namedtuple. The contents of ModelResult.inferences
         depends on the the specific inference type of this model, which
         can be queried by :meth:`.getInferenceType`.


Place the model in a permanent "finished learning" mode.
        In such a mode the model will not be able to learn from subsequent input
        records.

        .. note:: Upon completion of this command, learning may not be resumed on
           the given instance of the model (e.g., the implementation may optimize
           itself by pruning data structures that are necessary for learning).


Signal that the input record is the start of a new sequence. Normally called
to force the delineation of a sequence, such as between OPF tasks.


Return the sequence of :class:`~nupic.data.field_meta.FieldMetaInfo` objects
specifying the format of Model's output.

This may be different than the list of
:class:`~nupic.data.field_meta.FieldMetaInfo` objects supplied at
initialization (e.g., due to the transcoding of some input fields into
meta-fields, such as datetime -> dayOfWeek, timeOfDay, etc.).

:param includeClassifierOnlyField: (bool)
       If True, any field which is only sent to the classifier (i.e. not
       sent in to the bottom of the network) is also included
:returns: (list) of :class:`~nupic.data.field_meta.FieldMetaInfo` objects.


Propagate field statistics to the model in case some of its machinery
        needs it.

        :param fieldStats: (dict)
               A dict of dicts with first key being the fieldname and the second
               key is min,max or other supported statistics.


Get runtime statistics specific to this model, i.e.
        ``activeCellOverlapAvg``.

        :returns: (dict) A {statistic names: stats} dictionary


Get the logger for this object.
        This is a protected method that is used by the ModelBase to access the
        logger created by the subclass.

        :returns: (Logger) A Logger object, it should not be None.


return self.__inferenceType<EOL>

Return the InferenceType of this model.
        This is immutable.

        :returns: :class:`~nupic.frameworks.opf.opf_utils.InferenceType`


self.__learningEnabled = True<EOL>return<EOL>


self.__learningEnabled = False<EOL>return<EOL>

Turn Learning off for the current model.


return self.__learningEnabled<EOL>

Return the Learning state of the current model.

        :returns: (bool) The learning state


self.__inferenceEnabled = True<EOL>self.__inferenceArgs = inferenceArgs<EOL>

Enable inference for this model.

        :param inferenceArgs: (dict)
               A dictionary of arguments required for inference. These depend on
               the InferenceType of the current model


return self.__inferenceArgs<EOL>

Return the dict of arguments for the current inference mode.

        :returns: (dict) The arguments of the inference mode


self.__inferenceEnabled = False<EOL>

Turn Inference off for the current model.


return self.__inferenceEnabled<EOL>

Return the inference state of the current model.

        :returns: (bool) The inference state


raise NotImplementedError()<EOL>

Return the pycapnp proto type that the class uses for serialization.

        This is used to convert the proto into the proper type before passing it
        into the read or write method of the subclass.


path = os.path.join(checkpointDir, "<STR_LIT>")<EOL>path = os.path.abspath(path)<EOL>return path<EOL>

Return the absolute path of the model's checkpoint file.

        :param checkpointDir: (string)
               Directory of where the experiment is to be or was saved
        :returns: (string) An absolute path.


proto = self.getSchema().new_message()<EOL>self.write(proto)<EOL>checkpointPath = self._getModelCheckpointFilePath(checkpointDir)<EOL>if os.path.exists(checkpointDir):<EOL><INDENT>if not os.path.isdir(checkpointDir):<EOL><INDENT>raise Exception(("<STR_LIT>"<EOL>"<STR_LIT>")% checkpointDir)<EOL><DEDENT>if not os.path.isfile(checkpointPath):<EOL><INDENT>raise Exception(("<STR_LIT>"<EOL>"<STR_LIT>""<STR_LIT>") %(checkpointDir, checkpointPath))<EOL><DEDENT>shutil.rmtree(checkpointDir)<EOL><DEDENT>self.__makeDirectoryFromAbsolutePath(checkpointDir)<EOL>with open(checkpointPath, '<STR_LIT:wb>') as f:<EOL><INDENT>proto.write(f)<EOL><DEDENT>

Serializes model using capnproto and writes data to ``checkpointDir``


checkpointPath = cls._getModelCheckpointFilePath(checkpointDir)<EOL>with open(checkpointPath, '<STR_LIT:r>') as f:<EOL><INDENT>proto = cls.getSchema().read(f,<EOL>traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)<EOL><DEDENT>model = cls.read(proto)<EOL>return model<EOL>

Deserializes model from checkpointDir using capnproto


inferenceType = self.getInferenceType()<EOL>inferenceType = inferenceType[:<NUM_LIT:1>].lower() + inferenceType[<NUM_LIT:1>:]<EOL>proto.inferenceType = inferenceType<EOL>proto.numPredictions = self._numPredictions<EOL>proto.learningEnabled = self.__learningEnabled<EOL>proto.inferenceEnabled = self.__inferenceEnabled<EOL>proto.inferenceArgs = json.dumps(self.__inferenceArgs)<EOL>

Save the state maintained by the Model base class

        :param proto: capnp ModelProto message builder


raise NotImplementedError()<EOL>

Write state to proto object.

        The type of proto is determined by :meth:`getSchema`.


raise NotImplementedError()<EOL>

Read state from proto object.

        The type of proto is determined by :meth:`getSchema`.


logger = self._getLogger()<EOL>logger.debug("<STR_LIT>",<EOL>self, saveModelDir)<EOL>modelPickleFilePath = self._getModelPickleFilePath(saveModelDir)<EOL>if os.path.exists(saveModelDir):<EOL><INDENT>if not os.path.isdir(saveModelDir):<EOL><INDENT>raise Exception(("<STR_LIT>"<EOL>"<STR_LIT>")% saveModelDir)<EOL><DEDENT>if not os.path.isfile(modelPickleFilePath):<EOL><INDENT>raise Exception(("<STR_LIT>"<EOL>"<STR_LIT>""<STR_LIT>") %(saveModelDir, modelPickleFilePath))<EOL><DEDENT>shutil.rmtree(saveModelDir)<EOL><DEDENT>self.__makeDirectoryFromAbsolutePath(saveModelDir)<EOL>with open(modelPickleFilePath, '<STR_LIT:wb>') as modelPickleFile:<EOL><INDENT>logger.debug("<STR_LIT>", self)<EOL>pickle.dump(self, modelPickleFile, protocol=pickle.HIGHEST_PROTOCOL)<EOL>logger.debug("<STR_LIT>", self)<EOL><DEDENT>self._serializeExtraData(extraDataDir=self._getModelExtraDataDir(saveModelDir))<EOL>logger.debug("<STR_LIT>", self)<EOL>return<EOL>

Save the model in the given directory.

        :param saveModelDir: (string)
             Absolute directory path for saving the model. This directory should
             only be used to store a saved model. If the directory does not exist,
             it will be created automatically and populated with model data. A
             pre-existing directory will only be accepted if it contains previously
             saved model data. If such a directory is given, the full contents of
             the directory will be deleted and replaced with current model data.


pass<EOL>

Protected method that is called during serialization with an external
        directory path. It can be overridden by subclasses to bypass pickle for
        saving large binary states.
        This is called by ModelBase only.

        :param extraDataDir: (string) Model's extra data directory path


logger = opf_utils.initLogger(cls)<EOL>logger.debug("<STR_LIT>", savedModelDir)<EOL>modelPickleFilePath = Model._getModelPickleFilePath(savedModelDir)<EOL>with open(modelPickleFilePath, '<STR_LIT:rb>') as modelPickleFile:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>model = pickle.load(modelPickleFile)<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>model._deSerializeExtraData(<EOL>extraDataDir=Model._getModelExtraDataDir(savedModelDir))<EOL>logger.debug("<STR_LIT>")<EOL>return model<EOL>

Load saved model.

        :param savedModelDir: (string)
               Directory of where the experiment is to be or was saved
        :returns: (:class:`Model`) The loaded model instance


pass<EOL>

Protected method that is called during deserialization
        (after __setstate__) with an external directory path.
        It can be overridden by subclasses to bypass pickle for loading large
        binary states.
        This is called by ModelBase only.

        :param extraDataDir: (string) Model's extra data directory path


path = os.path.join(saveModelDir, "<STR_LIT>")<EOL>path = os.path.abspath(path)<EOL>return path<EOL>

Return the absolute path of the model's pickle file.

        :param saveModelDir: (string)
               Directory of where the experiment is to be or was saved
        :returns: (string) An absolute path.


Return the absolute path to the directory where the model's own
        "extra data" are stored (i.e., data that's too big for pickling).

        :param saveModelDir: (string)
               Directory of where the experiment is to be or was saved
        :returns: (string) An absolute path.


assert os.path.isabs(absDirPath)<EOL>try:<EOL><INDENT>os.makedirs(absDirPath)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != os.errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return<EOL>

Make directory for the given directory path if it doesn't already
        exist in the filesystem.

        :param absDirPath: (string) Absolute path of the directory to create
        @exception (Exception) OSError if directory creation fails


@wraps(func)<EOL>def _decorator(self, *args, **kwargs):<EOL><INDENT>if not self.getInferenceType() == InferenceType.TemporalAnomaly:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if self._getAnomalyClassifier() is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return func(self, *args, **kwargs)<EOL><DEDENT>return _decorator<EOL>

Decorator for functions that require anomaly models.


def __init__(self, net, statsCollectors):

self.net = net<EOL>self.statsCollectors = statsCollectors<EOL>return<EOL>

net:          The CLA Network instance
statsCollectors:
              Sequence of 0 or more CLAStatistic-based instances


if not inferenceType in self.__supportedInferenceKindSet:<EOL><INDENT>raise ValueError("<STR_LIT>".format(self.__class__, inferenceType))<EOL><DEDENT>if baseProto is None:<EOL><INDENT>super(HTMPredictionModel, self).__init__(inferenceType)<EOL><DEDENT>else:<EOL><INDENT>super(HTMPredictionModel, self).__init__(proto=baseProto)<EOL><DEDENT>self.__restoringFromState = False<EOL>self.__restoringFromV1 = False<EOL>self.__logger = initLogger(self)<EOL>self.__logger.debug("<STR_LIT>" % self.__myClassName)<EOL>self._minLikelihoodThreshold = minLikelihoodThreshold<EOL>self._maxPredictionsPerStep = maxPredictionsPerStep<EOL>self.__spLearningEnabled = bool(spEnable)<EOL>self.__tpLearningEnabled = bool(tmEnable)<EOL>if not InferenceType.isTemporal(self.getInferenceType())or self.getInferenceType() == InferenceType.NontemporalMultiStep:<EOL><INDENT>tmEnable = False<EOL><DEDENT>self._netInfo = None<EOL>self._hasSP = spEnable<EOL>self._hasTP = tmEnable<EOL>self._hasCL = clEnable<EOL>self._classifierInputEncoder = None<EOL>self._predictedFieldIdx = None<EOL>self._predictedFieldName = None<EOL>self._numFields = None<EOL>if network is not None:<EOL><INDENT>self._netInfo = NetworkInfo(net=network, statsCollectors=[])<EOL><DEDENT>else:<EOL><INDENT>self._netInfo = self.__createHTMNetwork(<EOL>sensorParams, spEnable, spParams, tmEnable, tmParams, clEnable,<EOL>clParams, anomalyParams)<EOL><DEDENT>if self.getInferenceType() == InferenceType.NontemporalAnomaly:<EOL><INDENT>self._getSPRegion().setParameter("<STR_LIT>", True)<EOL><DEDENT>if self.getInferenceType() == InferenceType.TemporalAnomaly:<EOL><INDENT>self._getTPRegion().setParameter("<STR_LIT>", True)<EOL><DEDENT>self.__trainSPNetOnlyIfRequested = trainSPNetOnlyIfRequested<EOL>self.__numRunCalls = <NUM_LIT:0><EOL>self.__finishedLearning = False<EOL>self.__logger.debug("<STR_LIT>" % self.__class__.__name__)<EOL>self._input = None<EOL>return<EOL>

:param network: if not None, the deserialized nupic.engine.Network instance
                to use instead of creating a new Network
:param baseProto: if not None, capnp ModelProto message reader for
                  deserializing; supersedes inferenceType


if paramName == '<STR_LIT>':<EOL><INDENT>return self.__numRunCalls<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(paramName))<EOL><DEDENT>

Currently only supports a parameter named ``__numRunCalls``.

:param paramName: (string) name of parameter to get. If not
       ``__numRunCalls`` an exception is thrown.
:returns: (int) the value of ``self.__numRunCalls``


@requireAnomalyModel<EOL><INDENT>def setAnomalyParameter(self, param, value):<DEDENT>

self._getAnomalyClassifier().setParameter(param, value)<EOL>

Set a parameter of the anomaly classifier within this model.

:param param: (string) name of parameter to set
:param value: (object) value to set


return self._getAnomalyClassifier().getParameter(param)<EOL>

Get a parameter of the anomaly classifier within this model by key.

:param param: (string) name of parameter to retrieve


@requireAnomalyModel<EOL><INDENT>def anomalyRemoveLabels(self, start, end, labelFilter):<DEDENT>

self._getAnomalyClassifier().getSelf().removeLabels(start, end, labelFilter)<EOL>

Remove labels from the anomaly classifier within this model. Removes all
records if ``labelFilter==None``, otherwise only removes the labels equal to
``labelFilter``.

:param start: (int) index to start removing labels
:param end: (int) index to end removing labels
:param labelFilter: (string) If specified, only removes records that match


@requireAnomalyModel<EOL><INDENT>def anomalyAddLabel(self, start, end, labelName):<DEDENT>

self._getAnomalyClassifier().getSelf().addLabel(start, end, labelName)<EOL>

Add labels from the anomaly classifier within this model.

:param start: (int) index to start label
:param end: (int) index to end label
:param labelName: (string) name of label


@requireAnomalyModel<EOL><INDENT>def anomalyGetLabels(self, start, end):<DEDENT>

return self._getAnomalyClassifier().getSelf().getLabels(start, end)<EOL>

Get labels from the anomaly classifier within this model.

:param start: (int) index to start getting labels
:param end: (int) index to end getting labels


sensor = self._getSensorRegion()<EOL>dataRow = copy.deepcopy(sensor.getSelf().getOutputValues('<STR_LIT>'))<EOL>dataDict = copy.deepcopy(inputRecord)<EOL>inputRecordEncodings = sensor.getSelf().getOutputValues('<STR_LIT>')<EOL>inputRecordCategory = int(sensor.getOutputData('<STR_LIT>')[<NUM_LIT:0>])<EOL>resetOut = sensor.getOutputData('<STR_LIT>')[<NUM_LIT:0>]<EOL>return SensorInput(dataRow=dataRow,<EOL>dataDict=dataDict,<EOL>dataEncodings=inputRecordEncodings,<EOL>sequenceReset=resetOut,<EOL>category=inputRecordCategory)<EOL>

inputRecord - dict containing the input to the sensor

Return a 'SensorInput' object, which represents the 'parsed'
representation of the input record


absoluteValue = None<EOL>bucketIdx = None<EOL>if self._predictedFieldName is not None and self._classifierInputEncoder is not None:<EOL><INDENT>absoluteValue = inputRecord[self._predictedFieldName]<EOL>bucketIdx = self._classifierInputEncoder.getBucketIndices(absoluteValue)[<NUM_LIT:0>]<EOL><DEDENT>return ClassifierInput(dataRow=absoluteValue,<EOL>bucketIndex=bucketIdx)<EOL>

inputRecord - dict containing the input to the sensor

Return a 'ClassifierInput' object, which contains the mapped
bucket index for input Record


inferenceType = self.getInferenceType()<EOL>inferences = {}<EOL>sp = self._getSPRegion()<EOL>score = None<EOL>if inferenceType == InferenceType.NontemporalAnomaly:<EOL><INDENT>score = sp.getOutputData("<STR_LIT>")[<NUM_LIT:0>] <EOL><DEDENT>elif inferenceType == InferenceType.TemporalAnomaly:<EOL><INDENT>tm = self._getTPRegion()<EOL>if sp is not None:<EOL><INDENT>activeColumns = sp.getOutputData("<STR_LIT>").nonzero()[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>sensor = self._getSensorRegion()<EOL>activeColumns = sensor.getOutputData('<STR_LIT>').nonzero()[<NUM_LIT:0>]<EOL><DEDENT>if not self._predictedFieldName in self._input:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>% self._predictedFieldName<EOL>)<EOL><DEDENT>score = tm.getOutputData("<STR_LIT>")[<NUM_LIT:0>]<EOL>if sp is not None:<EOL><INDENT>self._getAnomalyClassifier().setParameter(<EOL>"<STR_LIT>", len(activeColumns))<EOL>self._getAnomalyClassifier().prepareInputs()<EOL>self._getAnomalyClassifier().compute()<EOL>labels = self._getAnomalyClassifier().getSelf().getLabelResults()<EOL>inferences[InferenceElement.anomalyLabel] = "<STR_LIT:%s>" % labels<EOL><DEDENT><DEDENT>inferences[InferenceElement.anomalyScore] = score<EOL>return inferences<EOL>


inferenceArgs = self.getInferenceArgs()<EOL>predictedFieldName = inferenceArgs.get('<STR_LIT>', None)<EOL>if predictedFieldName is None:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>self._predictedFieldName = predictedFieldName<EOL>classifier = self._getClassifierRegion()<EOL>if not self._hasCL or classifier is None:<EOL><INDENT>return {}<EOL><DEDENT>sensor = self._getSensorRegion()<EOL>minLikelihoodThreshold = self._minLikelihoodThreshold<EOL>maxPredictionsPerStep = self._maxPredictionsPerStep<EOL>needLearning = self.isLearningEnabled()<EOL>inferences = {}<EOL>if self._classifierInputEncoder is None:<EOL><INDENT>if predictedFieldName is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>encoderList = sensor.getSelf().encoder.getEncoderList()<EOL>self._numFields = len(encoderList)<EOL>fieldNames = sensor.getSelf().encoder.getScalarNames()<EOL>if predictedFieldName in fieldNames:<EOL><INDENT>self._predictedFieldIdx = fieldNames.index(predictedFieldName)<EOL><DEDENT>else:<EOL><INDENT>self._predictedFieldIdx = None<EOL><DEDENT>if sensor.getSelf().disabledEncoder is not None:<EOL><INDENT>encoderList = sensor.getSelf().disabledEncoder.getEncoderList()<EOL><DEDENT>else:<EOL><INDENT>encoderList = []<EOL><DEDENT>if len(encoderList) >= <NUM_LIT:1>:<EOL><INDENT>fieldNames = sensor.getSelf().disabledEncoder.getScalarNames()<EOL>self._classifierInputEncoder = encoderList[fieldNames.index(<EOL>predictedFieldName)]<EOL><DEDENT>else:<EOL><INDENT>encoderList = sensor.getSelf().encoder.getEncoderList()<EOL>self._classifierInputEncoder = encoderList[self._predictedFieldIdx]<EOL><DEDENT><DEDENT>if not predictedFieldName in rawInput:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>% predictedFieldName)<EOL><DEDENT>absoluteValue = rawInput[predictedFieldName]<EOL>bucketIdx = self._classifierInputEncoder.getBucketIndices(absoluteValue)[<NUM_LIT:0>]<EOL>if isinstance(self._classifierInputEncoder, DeltaEncoder):<EOL><INDENT>if not hasattr(self,"<STR_LIT>"):<EOL><INDENT>self._ms_prevVal = absoluteValue<EOL><DEDENT>prevValue = self._ms_prevVal<EOL>self._ms_prevVal = absoluteValue<EOL>actualValue = absoluteValue - prevValue<EOL><DEDENT>else:<EOL><INDENT>actualValue = absoluteValue<EOL><DEDENT>if isinstance(actualValue, float) and math.isnan(actualValue):<EOL><INDENT>actualValue = SENTINEL_VALUE_FOR_MISSING_DATA<EOL><DEDENT>classifier.setParameter('<STR_LIT>', True)<EOL>classifier.setParameter('<STR_LIT>', needLearning)<EOL>classificationIn = {'<STR_LIT>': bucketIdx,<EOL>'<STR_LIT>': actualValue}<EOL>if inputTSRecordIdx is not None:<EOL><INDENT>recordNum = inputTSRecordIdx<EOL><DEDENT>else:<EOL><INDENT>recordNum = self.__numRunCalls<EOL><DEDENT>clResults = classifier.getSelf().customCompute(recordNum=recordNum,<EOL>patternNZ=patternNZ,<EOL>classification=classificationIn)<EOL>predictionSteps = classifier.getParameter('<STR_LIT>')<EOL>predictionSteps = [int(x) for x in predictionSteps.split('<STR_LIT:U+002C>')]<EOL>inferences[InferenceElement.multiStepPredictions] = dict()<EOL>inferences[InferenceElement.multiStepBestPredictions] = dict()<EOL>inferences[InferenceElement.multiStepBucketLikelihoods] = dict()<EOL>for steps in predictionSteps:<EOL><INDENT>likelihoodsVec = clResults[steps]<EOL>bucketValues = clResults['<STR_LIT>']<EOL>likelihoodsDict = dict()<EOL>bestActValue = None<EOL>bestProb = None<EOL>for (actValue, prob) in zip(bucketValues, likelihoodsVec):<EOL><INDENT>if actValue in likelihoodsDict:<EOL><INDENT>likelihoodsDict[actValue] += prob<EOL><DEDENT>else:<EOL><INDENT>likelihoodsDict[actValue] = prob<EOL><DEDENT>if bestProb is None or likelihoodsDict[actValue] > bestProb:<EOL><INDENT>bestProb = likelihoodsDict[actValue]<EOL>bestActValue = actValue<EOL><DEDENT><DEDENT>likelihoodsDict = HTMPredictionModel._removeUnlikelyPredictions(<EOL>likelihoodsDict, minLikelihoodThreshold, maxPredictionsPerStep)<EOL>bucketLikelihood = {}<EOL>for k in likelihoodsDict.keys():<EOL><INDENT>bucketLikelihood[self._classifierInputEncoder.getBucketIndices(k)[<NUM_LIT:0>]] = (<EOL>likelihoodsDict[k])<EOL><DEDENT>if isinstance(self._classifierInputEncoder, DeltaEncoder):<EOL><INDENT>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._ms_predHistories = dict()<EOL><DEDENT>predHistories = self._ms_predHistories<EOL>if not steps in predHistories:<EOL><INDENT>predHistories[steps] = deque()<EOL><DEDENT>predHistory = predHistories[steps]<EOL>sumDelta = sum(predHistory)<EOL>offsetDict = dict()<EOL>for (k, v) in likelihoodsDict.iteritems():<EOL><INDENT>if k is not None:<EOL><INDENT>offsetDict[absoluteValue+float(k)+sumDelta] = v<EOL><DEDENT><DEDENT>bucketLikelihoodOffset = {}<EOL>for k in offsetDict.keys():<EOL><INDENT>bucketLikelihoodOffset[self._classifierInputEncoder.getBucketIndices(k)[<NUM_LIT:0>]] = (<EOL>offsetDict[k])<EOL><DEDENT>if bestActValue is not None:<EOL><INDENT>predHistory.append(bestActValue)<EOL><DEDENT>if len(predHistory) >= steps:<EOL><INDENT>predHistory.popleft()<EOL><DEDENT>if len(offsetDict)><NUM_LIT:0>:<EOL><INDENT>inferences[InferenceElement.multiStepPredictions][steps] = offsetDict<EOL>inferences[InferenceElement.multiStepBucketLikelihoods][steps] = bucketLikelihoodOffset<EOL><DEDENT>else:<EOL><INDENT>inferences[InferenceElement.multiStepPredictions][steps] = likelihoodsDict<EOL>inferences[InferenceElement.multiStepBucketLikelihoods][steps] = bucketLikelihood<EOL><DEDENT>if bestActValue is None:<EOL><INDENT>inferences[InferenceElement.multiStepBestPredictions][steps] = None<EOL><DEDENT>else:<EOL><INDENT>inferences[InferenceElement.multiStepBestPredictions][steps] = (<EOL>absoluteValue + sumDelta + bestActValue)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>inferences[InferenceElement.multiStepPredictions][steps] = (<EOL>likelihoodsDict)<EOL>inferences[InferenceElement.multiStepBestPredictions][steps] = (<EOL>bestActValue)<EOL>inferences[InferenceElement.multiStepBucketLikelihoods][steps] = (<EOL>bucketLikelihood)<EOL><DEDENT><DEDENT>return inferences<EOL>

Handle the CLA Classifier compute logic when implementing multi-step
        prediction. This is where the patternNZ is associated with one of the
        other fields from the dataset 0 to N steps in the future. This method is
        used by each type of network (encoder only, SP only, SP +TM) to handle the
        compute logic through the CLA Classifier. It fills in the inference dict with
        the results of the compute.

        Parameters:
        -------------------------------------------------------------------
        patternNZ: The input to the CLA Classifier as a list of active input indices
        inputTSRecordIdx: The index of the record as computed from the timestamp
                      and aggregation interval. This normally increments by 1
                      each time unless there are missing records. If there is no
                      aggregation interval or timestamp in the data, this will be
                      None.
        rawInput:   The raw input to the sensor, as a dict.


maxVal = (None, None)<EOL>for (k, v) in likelihoodsDict.items():<EOL><INDENT>if len(likelihoodsDict) <= <NUM_LIT:1>:<EOL><INDENT>break<EOL><DEDENT>if maxVal[<NUM_LIT:0>] is None or v >= maxVal[<NUM_LIT:1>]:<EOL><INDENT>if maxVal[<NUM_LIT:0>] is not None and maxVal[<NUM_LIT:1>] < minLikelihoodThreshold:<EOL><INDENT>del likelihoodsDict[maxVal[<NUM_LIT:0>]]<EOL><DEDENT>maxVal = (k, v)<EOL><DEDENT>elif v < minLikelihoodThreshold:<EOL><INDENT>del likelihoodsDict[k]<EOL><DEDENT><DEDENT>likelihoodsDict = dict(sorted(likelihoodsDict.iteritems(),<EOL>key=itemgetter(<NUM_LIT:1>),<EOL>reverse=True)[:maxPredictionsPerStep])<EOL>return likelihoodsDict<EOL>

Remove entries with 0 likelihood or likelihood less than
        minLikelihoodThreshold, but don't leave an empty dict.


ret = {"<STR_LIT>" : self.__numRunCalls}<EOL>temporalStats = dict()<EOL>if self._hasTP:<EOL><INDENT>for stat in self._netInfo.statsCollectors:<EOL><INDENT>sdict = stat.getStats()<EOL>temporalStats.update(sdict)<EOL><DEDENT><DEDENT>ret[InferenceType.getLabel(InferenceType.TemporalNextStep)] = temporalStats<EOL>return ret<EOL>

Only returns data for a stat called ``numRunCalls``.
:return:


return self.__logger<EOL>

Get the logger for this object. This is a protected method that is used
        by the Model to access the logger created by the subclass

        return:
          A logging.Logger object. Should not be None


return self._netInfo.net.regions.get('<STR_LIT>', None)<EOL>

Returns reference to the network's SP region


Returns reference to the network's TM region


return self._netInfo.net.regions['<STR_LIT>']<EOL>

Returns reference to the network's Sensor region


if (self._netInfo.net is not None and<EOL>"<STR_LIT>" in self._netInfo.net.regions):<EOL><INDENT>return self._netInfo.net.regions["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Returns reference to the network's Classifier region


return  self._getSensorRegion().getSelf().encoder<EOL>

Returns:  sensor region's encoder for the given network


return  self._getSensorRegion().getSelf().disabledEncoder<EOL>

Returns:  sensor region's encoder that is sent only to the classifier,
            not to the bottom of the network


return self._getSensorRegion().getSelf().dataSource<EOL>

Returns: data source that we installed in sensor region


def __createHTMNetwork(self, sensorParams, spEnable, spParams, tmEnable,<EOL>tmParams, clEnable, clParams, anomalyParams):

<EOL>n = Network()<EOL>n.addRegion("<STR_LIT>", "<STR_LIT>", json.dumps(dict(verbosity=sensorParams['<STR_LIT>'])))<EOL>sensor = n.regions['<STR_LIT>'].getSelf()<EOL>enabledEncoders = copy.deepcopy(sensorParams['<STR_LIT>'])<EOL>for name, params in enabledEncoders.items():<EOL><INDENT>if params is not None:<EOL><INDENT>classifierOnly = params.pop('<STR_LIT>', False)<EOL>if classifierOnly:<EOL><INDENT>enabledEncoders.pop(name)<EOL><DEDENT><DEDENT><DEDENT>disabledEncoders = copy.deepcopy(sensorParams['<STR_LIT>'])<EOL>for name, params in disabledEncoders.items():<EOL><INDENT>if params is None:<EOL><INDENT>disabledEncoders.pop(name)<EOL><DEDENT>else:<EOL><INDENT>classifierOnly = params.pop('<STR_LIT>', False)<EOL>if not classifierOnly:<EOL><INDENT>disabledEncoders.pop(name)<EOL><DEDENT><DEDENT><DEDENT>encoder = MultiEncoder(enabledEncoders)<EOL>sensor.encoder = encoder<EOL>sensor.disabledEncoder = MultiEncoder(disabledEncoders)<EOL>sensor.dataSource = DataBuffer()<EOL>prevRegion = "<STR_LIT>"<EOL>prevRegionWidth = encoder.getWidth()<EOL>if spEnable:<EOL><INDENT>spParams = spParams.copy()<EOL>spParams['<STR_LIT>'] = prevRegionWidth<EOL>self.__logger.debug("<STR_LIT>" % spParams)<EOL>n.addRegion("<STR_LIT>", "<STR_LIT>", json.dumps(spParams))<EOL>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL>prevRegion = "<STR_LIT>"<EOL>prevRegionWidth = spParams['<STR_LIT>']<EOL><DEDENT>if tmEnable:<EOL><INDENT>tmParams = tmParams.copy()<EOL>if prevRegion == '<STR_LIT>':<EOL><INDENT>tmParams['<STR_LIT>'] = tmParams['<STR_LIT>'] = prevRegionWidth<EOL><DEDENT>else:<EOL><INDENT>assert tmParams['<STR_LIT>'] == prevRegionWidth<EOL>tmParams['<STR_LIT>'] = tmParams['<STR_LIT>']<EOL><DEDENT>self.__logger.debug("<STR_LIT>" % tmParams)<EOL>n.addRegion("<STR_LIT>", "<STR_LIT>", json.dumps(tmParams))<EOL>n.link(prevRegion, "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>if prevRegion != "<STR_LIT>":<EOL><INDENT>n.link("<STR_LIT>", prevRegion, "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>n.link("<STR_LIT>", prevRegion, "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL><DEDENT>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL>prevRegion = "<STR_LIT>"<EOL>prevRegionWidth = tmParams['<STR_LIT>']<EOL><DEDENT>if clEnable and clParams is not None:<EOL><INDENT>clParams = clParams.copy()<EOL>clRegionName = clParams.pop('<STR_LIT>')<EOL>self.__logger.debug("<STR_LIT>" % (clRegionName,<EOL>clParams))<EOL>n.addRegion("<STR_LIT>", "<STR_LIT>" % str(clRegionName), json.dumps(clParams))<EOL>if str(clRegionName) == "<STR_LIT>":<EOL><INDENT>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL><DEDENT>n.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", srcOutput="<STR_LIT>",<EOL>destInput="<STR_LIT>")<EOL>n.link(prevRegion, "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL><DEDENT>if self.getInferenceType() == InferenceType.TemporalAnomaly:<EOL><INDENT>anomalyClParams = dict(<EOL>trainRecords=anomalyParams.get('<STR_LIT>', None),<EOL>cacheSize=anomalyParams.get('<STR_LIT>', None)<EOL>)<EOL>self._addAnomalyClassifierRegion(n, anomalyClParams, spEnable, tmEnable)<EOL><DEDENT>n.initialize()<EOL>return NetworkInfo(net=n, statsCollectors=[])<EOL>

Create a CLA network and return it.

        description:  HTMPredictionModel description dictionary (TODO: define schema)
        Returns:      NetworkInfo instance;


<EOL>state = self.__dict__.copy()<EOL>state["<STR_LIT>"] = NetworkInfo(net=None,<EOL>statsCollectors=self._netInfo.statsCollectors)<EOL>for ephemeral in [self.__manglePrivateMemberName("<STR_LIT>"),<EOL>self.__manglePrivateMemberName("<STR_LIT>")]:<EOL><INDENT>state.pop(ephemeral)<EOL><DEDENT>return state<EOL>

Return serializable state.  This function will return a version of the
__dict__ with data that shouldn't be pickled stripped out. In particular,
the CLA Network is stripped out because it has it's own serialization
mechanism)

See also: _serializeExtraData()


self.__dict__.update(state)<EOL>self.__restoringFromState = True<EOL>self.__logger = initLogger(self)<EOL>if not hasattr(self, "<STR_LIT>"):<EOL><INDENT>self.__restoringFromV1 = True<EOL>self._hasSP = True<EOL>if self.__temporalNetInfo is not None:<EOL><INDENT>self._Model__inferenceType = InferenceType.TemporalNextStep<EOL>self._netInfo = self.__temporalNetInfo<EOL>self._hasTP = True<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>self._Model__inferenceArgs = {}<EOL>self._Model__learningEnabled = True<EOL>self._Model__inferenceEnabled = True<EOL>self.__dict__.pop("<STR_LIT>", None)<EOL>self.__dict__.pop("<STR_LIT>", None)<EOL>self.__dict__.pop("<STR_LIT>", None)<EOL><DEDENT>if not hasattr(self, "<STR_LIT>"):<EOL><INDENT>self._hasSP = False<EOL>self._hasTP = False<EOL>if self.__encoderNetInfo is not None:<EOL><INDENT>self._netInfo = self.__encoderNetInfo<EOL><DEDENT>elif self.__nonTemporalNetInfo is not None:<EOL><INDENT>self._netInfo = self.__nonTemporalNetInfo<EOL>self._hasSP = True<EOL><DEDENT>else:<EOL><INDENT>self._netInfo = self.__temporalNetInfo<EOL>self._hasSP = True<EOL>self._hasTP = True<EOL><DEDENT>self.__dict__.pop("<STR_LIT>", None)<EOL>self.__dict__.pop("<STR_LIT>", None)<EOL>self.__dict__.pop("<STR_LIT>", None)<EOL><DEDENT>self._classifierInputEncoder = None<EOL>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._minLikelihoodThreshold = DEFAULT_LIKELIHOOD_THRESHOLD<EOL><DEDENT>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._maxPredictionsPerStep = DEFAULT_MAX_PREDICTIONS_PER_STEP<EOL><DEDENT>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._hasCL = (self._getClassifierRegion() is not None)<EOL><DEDENT>self.__logger.debug("<STR_LIT>" % self.__class__.__name__)<EOL>

Set the state of ourself from a serialized state.

See also: _deSerializeExtraData


super(HTMPredictionModel, self).writeBaseToProto(proto.modelBase)<EOL>proto.numRunCalls = self.__numRunCalls<EOL>proto.minLikelihoodThreshold = self._minLikelihoodThreshold<EOL>proto.maxPredictionsPerStep = self._maxPredictionsPerStep<EOL>self._netInfo.net.write(proto.network)<EOL>proto.spLearningEnabled = self.__spLearningEnabled<EOL>proto.tpLearningEnabled = self.__tpLearningEnabled<EOL>if self._predictedFieldIdx is None:<EOL><INDENT>proto.predictedFieldIdx.none = None<EOL><DEDENT>else:<EOL><INDENT>proto.predictedFieldIdx.value = self._predictedFieldIdx<EOL><DEDENT>if self._predictedFieldName is None:<EOL><INDENT>proto.predictedFieldName.none = None<EOL><DEDENT>else:<EOL><INDENT>proto.predictedFieldName.value = self._predictedFieldName<EOL><DEDENT>if self._numFields is None:<EOL><INDENT>proto.numFields.none = None<EOL><DEDENT>else:<EOL><INDENT>proto.numFields.value = self._numFields<EOL><DEDENT>proto.trainSPNetOnlyIfRequested = self.__trainSPNetOnlyIfRequested<EOL>proto.finishedLearning = self.__finishedLearning<EOL>

:param proto: capnp HTMPredictionModelProto message builder


obj = object.__new__(cls)<EOL>super(HTMPredictionModel, obj).__init__(proto=proto.modelBase)<EOL>obj._minLikelihoodThreshold = round(proto.minLikelihoodThreshold,<EOL>EPSILON_ROUND)<EOL>obj._maxPredictionsPerStep = proto.maxPredictionsPerStep<EOL>network = Network.read(proto.network)<EOL>obj._hasSP = ("<STR_LIT>" in network.regions)<EOL>obj._hasTP = ("<STR_LIT>" in network.regions)<EOL>obj._hasCL = ("<STR_LIT>" in network.regions)<EOL>obj._netInfo = NetworkInfo(net=network, statsCollectors=[])<EOL>obj.__spLearningEnabled = bool(proto.spLearningEnabled)<EOL>obj.__tpLearningEnabled = bool(proto.tpLearningEnabled)<EOL>obj.__numRunCalls = proto.numRunCalls<EOL>obj._classifierInputEncoder = None<EOL>if proto.predictedFieldIdx.which() == "<STR_LIT:none>":<EOL><INDENT>obj._predictedFieldIdx = None<EOL><DEDENT>else:<EOL><INDENT>obj._predictedFieldIdx = proto.predictedFieldIdx.value<EOL><DEDENT>if proto.predictedFieldName.which() == "<STR_LIT:none>":<EOL><INDENT>obj._predictedFieldName = None<EOL><DEDENT>else:<EOL><INDENT>obj._predictedFieldName = proto.predictedFieldName.value<EOL><DEDENT>obj._numFields = proto.numFields<EOL>if proto.numFields.which() == "<STR_LIT:none>":<EOL><INDENT>obj._numFields = None<EOL><DEDENT>else:<EOL><INDENT>obj._numFields = proto.numFields.value<EOL><DEDENT>obj.__trainSPNetOnlyIfRequested = proto.trainSPNetOnlyIfRequested<EOL>obj.__finishedLearning = proto.finishedLearning<EOL>obj._input = None<EOL>sensor = network.regions['<STR_LIT>'].getSelf()<EOL>sensor.dataSource = DataBuffer()<EOL>network.initialize()<EOL>obj.__logger = initLogger(obj)<EOL>obj.__logger.debug("<STR_LIT>" % obj.__myClassName)<EOL>obj.__restoringFromState = False<EOL>obj.__restoringFromV1 = False<EOL>return obj<EOL>

:param proto: capnp HTMPredictionModelProto message reader


[virtual method override] This method is called during serialization
        with an external directory path that can be used to bypass pickle for saving
        large binary states.

        extraDataDir:
                      Model's extra data directory path


assert self.__restoringFromState<EOL>assert (self._netInfo.net is None), "<STR_LIT>"<EOL>stateDir = self.__getNetworkStateDirectory(extraDataDir=extraDataDir)<EOL>self.__logger.debug(<EOL>"<STR_LIT>", self)<EOL>self._netInfo.net = Network(stateDir)<EOL>self.__logger.debug(<EOL>"<STR_LIT>", self)<EOL>self._netInfo.net.initialize()<EOL>if self.getInferenceType() == InferenceType.TemporalAnomaly:<EOL><INDENT>classifierType = self._getAnomalyClassifier().getSelf().__class__.__name__<EOL>if classifierType is '<STR_LIT>':<EOL><INDENT>anomalyClParams = dict(<EOL>trainRecords=self._classifier_helper._autoDetectWaitRecords,<EOL>cacheSize=self._classifier_helper._history_length,<EOL>)<EOL>spEnable = (self._getSPRegion() is not None)<EOL>tmEnable = True<EOL>knnRegion = self._getAnomalyClassifier().getSelf()<EOL>self._addAnomalyClassifierRegion(self._netInfo.net, anomalyClParams,<EOL>spEnable, tmEnable)<EOL>self._getAnomalyClassifier().getSelf()._iteration = self.__numRunCalls<EOL>self._getAnomalyClassifier().getSelf()._recordsCache = (<EOL>self._classifier_helper.saved_states)<EOL>self._getAnomalyClassifier().getSelf().saved_categories = (<EOL>self._classifier_helper.saved_categories)<EOL>self._getAnomalyClassifier().getSelf()._knnclassifier = knnRegion<EOL>self._getTPRegion().setParameter('<STR_LIT>', True)<EOL>del self._classifier_helper<EOL>self._netInfo.net.initialize()<EOL><DEDENT><DEDENT>self.__restoringFromState = False<EOL>self.__logger.debug("<STR_LIT>", self)<EOL>return<EOL>

[virtual method override] This method is called during deserialization
        (after __setstate__) with an external directory path that can be used to
        bypass pickle for loading large binary states.

        extraDataDir:
                      Model's extra data directory path


def _addAnomalyClassifierRegion(self, network, params, spEnable, tmEnable):

allParams = copy.deepcopy(params)<EOL>knnParams = dict(k=<NUM_LIT:1>,<EOL>distanceMethod='<STR_LIT>',<EOL>distanceNorm=<NUM_LIT:1>,<EOL>doBinarization=<NUM_LIT:1>,<EOL>replaceDuplicates=<NUM_LIT:0>,<EOL>maxStoredPatterns=<NUM_LIT:1000>)<EOL>allParams.update(knnParams)<EOL>if allParams['<STR_LIT>'] is None:<EOL><INDENT>allParams['<STR_LIT>'] = DEFAULT_ANOMALY_TRAINRECORDS<EOL><DEDENT>if allParams['<STR_LIT>'] is None:<EOL><INDENT>allParams['<STR_LIT>'] = DEFAULT_ANOMALY_CACHESIZE<EOL><DEDENT>if self._netInfo is not None and self._netInfo.net is not Noneand self._getAnomalyClassifier() is not None:<EOL><INDENT>self._netInfo.net.removeRegion('<STR_LIT>')<EOL><DEDENT>network.addRegion("<STR_LIT>",<EOL>"<STR_LIT>",<EOL>json.dumps(allParams))<EOL>if spEnable:<EOL><INDENT>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL><DEDENT>if tmEnable:<EOL><INDENT>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>

Attaches an 'AnomalyClassifier' region to the network. Will remove current
'AnomalyClassifier' region if it exists.

Parameters
-----------
network - network to add the AnomalyClassifier region
params - parameters to pass to the region
spEnable - True if network has an SP region
tmEnable - True if network has a TM region; Currently requires True


if self.__restoringFromV1:<EOL><INDENT>if self.getInferenceType() == InferenceType.TemporalNextStep:<EOL><INDENT>leafName = '<STR_LIT>'+ "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>leafName = '<STR_LIT>'+ "<STR_LIT>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>leafName = InferenceType.getLabel(self.getInferenceType()) + "<STR_LIT>"<EOL><DEDENT>path = os.path.join(extraDataDir, leafName)<EOL>path = os.path.abspath(path)<EOL>return path<EOL>

extraDataDir:
              Model's extra data directory path
Returns:      Absolute directory path for saving CLA Network


def __manglePrivateMemberName(self, privateMemberName, skipCheck=False):

assert privateMemberName.startswith("<STR_LIT>"),"<STR_LIT>" % privateMemberName<EOL>assert not privateMemberName.startswith("<STR_LIT>"),"<STR_LIT>" % privateMemberName<EOL>assert not privateMemberName.endswith("<STR_LIT>"),"<STR_LIT>" % privateMemberName<EOL>realName = "<STR_LIT:_>" + (self.__myClassName).lstrip("<STR_LIT:_>") + privateMemberName<EOL>if not skipCheck:<EOL><INDENT>getattr(self, realName)<EOL><DEDENT>return realName<EOL>

Mangles the given mangled (private) member name; a mangled member name
        is one whose name begins with two or more underscores and ends with one
        or zero underscores.

        privateMemberName:
                      The private member name (e.g., "__logger")

        skipCheck:    Pass True to skip test for presence of the demangled member
                      in our instance.

        Returns:      The demangled member name (e.g., "_HTMPredictionModel__logger")


Emits periodic metrics to stdout in JSON.

        :param metrics: A list of metrics as returned by
              :meth:`nupic.frameworks.opf.opf_task_driver.OPFTaskDriver.getMetrics`.


Emits final metrics.

        .. note:: the intention is that the final metrics may go to a different
                  place (e.g., csv file) versus :meth:`emitPeriodicMetrics`
                  (e.g., stdout)

        :param metrics: A list of metrics as returned by
              :meth:`nupic.frameworks.opf.opf_task_driver.OPFTaskDriver.getMetrics`.


:returns:     a tuple of dataset field metadata descriptors that are
              arranged in the same order as the columns in the dataset.
              Each field metadata descriptor is of type
              :class:`nupic.data.field_meta.FieldMetaInfo`


:returns:     The next record from the dataset.  The returned record object
              is of the same structure as returned by
              :meth:`nupic.data.record_stream.RecordStreamIface.getNextRecord`.
              Returns ``None`` if the next record is not available yet.

:raises: (StopIteration) if a hard "end of file" has been reached
              and no more records will be forthcoming.


Closes the writer (e.g., close the underlying file)


@abstractmethod<EOL><INDENT>def append(self, inputRow, predictionRow, sequenceReset, metrics=None):<DEDENT>

Emits a single prediction as input versus predicted.

        inputRow:       A tuple or list of fields comprising the input data row.
        predictionRow:  A tuple or list of fields comprising the prediction, or None
                        if prediction is not available.  The None use case is
                        intended for temporal inference where there is no matching
                        prediction for the same timestep as the given ground truth,
                        such as the case with the very first input record.
        sequenceReset:  A value that tests True if the input row was
                        accompanied by a sequence reset signal; False if not
                        accompanied by a sequence reset signal.

        metrics:        OPTIONAL -A dictionary of metrics that will be written out
                        with every prediction. The keys are the automatically
                        generated metric labels (see MetricSpec in
                        prediction_metrics_manager.py), and the value is the real
                        number value of the metric.


@abstractmethod<EOL><INDENT>def checkpoint(self, checkpointSink, maxRows):<DEDENT>

Save a checkpoint of the prediction output stream. The checkpoint
        comprises up to maxRows of the most recent inference records.

        Parameters:
        ----------------------------------------------------------------------
        checkpointSink:     A File-like object where predictions checkpoint data, if
                            any, will be stored.
        maxRows:            Maximum number of most recent inference rows
                            to checkpoint.


def _translateMetricsToJSON(self, metrics, label):

<EOL>metricsDict = metrics<EOL>def _mapNumpyValues(obj):<EOL><INDENT>"""<STR_LIT:U+0020>"""<EOL>import numpy<EOL>if isinstance(obj, numpy.float32):<EOL><INDENT>return float(obj)<EOL><DEDENT>elif isinstance(obj, numpy.bool_):<EOL><INDENT>return bool(obj)<EOL><DEDENT>elif isinstance(obj, numpy.ndarray):<EOL><INDENT>return obj.tolist()<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % (obj, obj.__class__))<EOL><DEDENT><DEDENT>jsonString = json.dumps(metricsDict, indent=<NUM_LIT:4>, default=_mapNumpyValues)<EOL>return jsonString<EOL>

Translates the given metrics value to JSON string

        metrics:        A list of dictionaries per OPFTaskDriver.getMetrics():

        Returns:        JSON string representing the given metrics object.


def __init__(self, experimentDir, label, inferenceType,<EOL>fields, metricNames=None, checkpointSource=None):

<EOL>self.__experimentDir = experimentDir<EOL>self.__inferenceType = inferenceType<EOL>self.__inputFieldsMeta = tuple(copy.deepcopy(fields))<EOL>self.__numInputFields = len(self.__inputFieldsMeta)<EOL>self.__label = label<EOL>if metricNames is not None:<EOL><INDENT>metricNames.sort()<EOL><DEDENT>self.__metricNames = metricNames<EOL>self.__outputFieldsMeta = []<EOL>self._rawInputNames = []<EOL>self.__datasetPath = None<EOL>self.__dataset = None<EOL>self.__checkpointCache = None<EOL>if checkpointSource is not None:<EOL><INDENT>checkpointSource.seek(<NUM_LIT:0>)<EOL>self.__checkpointCache = io.StringIO()<EOL>shutil.copyfileobj(checkpointSource, self.__checkpointCache)<EOL><DEDENT>return<EOL>

Constructor

        experimentDir:
                      experiment directory path that contains description.py

        label:        A label string to incorporate into the filename.

        inferenceElements:

        inferenceType:
                      An constant from opf_utils.InferenceType for the
                      requested prediction writer

        fields:       a non-empty sequence of nupic.data.fieldmeta.FieldMetaInfo
                      representing fields that will be emitted to this prediction
                      writer

        metricNames:  OPTIONAL - A list of metric names that well be emiited by this
                      prediction writer

        checkpointSource:
                      If not None, a File-like object containing the
                      previously-checkpointed predictions for setting the initial
                      contents of this PredictionOutputStream.  Will be copied
                      before returning, if needed.


<EOL>resetFieldMeta = FieldMetaInfo(<EOL>name="<STR_LIT>",<EOL>type=FieldMetaType.integer,<EOL>special = FieldMetaSpecial.reset)<EOL>self.__outputFieldsMeta.append(resetFieldMeta)<EOL>rawInput = modelResult.rawInput<EOL>rawFields = list(rawInput.keys())<EOL>rawFields.sort()<EOL>for field in rawFields:<EOL><INDENT>if field.startswith('<STR_LIT:_>') or field == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>value = rawInput[field]<EOL>meta = FieldMetaInfo(name=field, type=FieldMetaType.string,<EOL>special=FieldMetaSpecial.none)<EOL>self.__outputFieldsMeta.append(meta)<EOL>self._rawInputNames.append(field)<EOL><DEDENT>for inferenceElement, value in modelResult.inferences.items():<EOL><INDENT>inferenceLabel = InferenceElement.getLabel(inferenceElement)<EOL>if type(value) in (list, tuple):<EOL><INDENT>self.__outputFieldsMeta.extend(self.__getListMetaInfo(inferenceElement))<EOL><DEDENT>elif isinstance(value, dict):<EOL><INDENT>self.__outputFieldsMeta.extend(self.__getDictMetaInfo(inferenceElement,<EOL>value))<EOL><DEDENT>else:<EOL><INDENT>if InferenceElement.getInputElement(inferenceElement):<EOL><INDENT>self.__outputFieldsMeta.append(FieldMetaInfo(name=inferenceLabel+"<STR_LIT>",<EOL>type=FieldMetaType.string, special = '<STR_LIT>'))<EOL><DEDENT>self.__outputFieldsMeta.append(FieldMetaInfo(name=inferenceLabel,<EOL>type=FieldMetaType.string, special = '<STR_LIT>'))<EOL><DEDENT><DEDENT>if self.__metricNames:<EOL><INDENT>for metricName in self.__metricNames:<EOL><INDENT>metricField = FieldMetaInfo(<EOL>name = metricName,<EOL>type = FieldMetaType.float,<EOL>special = FieldMetaSpecial.none)<EOL>self.__outputFieldsMeta.append(metricField)<EOL><DEDENT><DEDENT>inferenceDir = _FileUtils.createExperimentInferenceDir(self.__experimentDir)<EOL>filename = (self.__label + "<STR_LIT:.>" +<EOL>opf_utils.InferenceType.getLabel(self.__inferenceType) +<EOL>"<STR_LIT>")<EOL>self.__datasetPath = os.path.join(inferenceDir, filename)<EOL>print("<STR_LIT>" % self.__datasetPath)<EOL>print("<STR_LIT>" % ([tuple(i) for i in self.__outputFieldsMeta],))<EOL>self.__dataset = FileRecordStream(streamID=self.__datasetPath, write=True,<EOL>fields=self.__outputFieldsMeta)<EOL>if self.__checkpointCache is not None:<EOL><INDENT>self.__checkpointCache.seek(<NUM_LIT:0>)<EOL>reader = csv.reader(self.__checkpointCache, dialect='<STR_LIT>')<EOL>try:<EOL><INDENT>header = next(reader)<EOL><DEDENT>except StopIteration:<EOL><INDENT>print("<STR_LIT>" % (self.__datasetPath,))<EOL><DEDENT>else:<EOL><INDENT>assert tuple(self.__dataset.getFieldNames()) == tuple(header),"<STR_LIT>" % (<EOL>tuple(self.__dataset.getFieldNames()), tuple(header))<EOL><DEDENT>numRowsCopied = <NUM_LIT:0><EOL>while True:<EOL><INDENT>try:<EOL><INDENT>row = next(reader)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>self.__dataset.appendRecord(row)<EOL>numRowsCopied += <NUM_LIT:1><EOL><DEDENT>self.__dataset.flush()<EOL>print("<STR_LIT>" % (<EOL>numRowsCopied, self.__datasetPath))<EOL>self.__checkpointCache.close()<EOL>self.__checkpointCache = None<EOL><DEDENT>return<EOL>

Open the data file and write the header row


if metricNames is None:<EOL><INDENT>self.__metricNames = set([])<EOL><DEDENT>else:<EOL><INDENT>self.__metricNames = set(metricNames)<EOL><DEDENT>

Tell the writer which metrics should be written

        Parameters:
        -----------------------------------------------------------------------
        metricsNames: A list of metric lables to be written


if self.__dataset:<EOL><INDENT>self.__dataset.close()<EOL><DEDENT>self.__dataset = None<EOL>return<EOL>

[virtual method override] Closes the writer (e.g., close the underlying
        file)


fieldMetaInfo = []<EOL>inferenceLabel = InferenceElement.getLabel(inferenceElement)<EOL>for inputFieldMeta in self.__inputFieldsMeta:<EOL><INDENT>if InferenceElement.getInputElement(inferenceElement):<EOL><INDENT>outputFieldMeta = FieldMetaInfo(<EOL>name=inputFieldMeta.name + "<STR_LIT>",<EOL>type=inputFieldMeta.type,<EOL>special=inputFieldMeta.special<EOL>)<EOL><DEDENT>predictionField = FieldMetaInfo(<EOL>name=inputFieldMeta.name + "<STR_LIT:.>" + inferenceLabel,<EOL>type=inputFieldMeta.type,<EOL>special=inputFieldMeta.special<EOL>)<EOL>fieldMetaInfo.append(outputFieldMeta)<EOL>fieldMetaInfo.append(predictionField)<EOL><DEDENT>return fieldMetaInfo<EOL>

Get field metadata information for inferences that are of list type
        TODO: Right now we assume list inferences are associated with the input field
        metadata


def __getDictMetaInfo(self, inferenceElement, inferenceDict):

fieldMetaInfo = []<EOL>inferenceLabel = InferenceElement.getLabel(inferenceElement)<EOL>if InferenceElement.getInputElement(inferenceElement):<EOL><INDENT>fieldMetaInfo.append(FieldMetaInfo(name=inferenceLabel+"<STR_LIT>",<EOL>type=FieldMetaType.string,<EOL>special = '<STR_LIT>'))<EOL><DEDENT>keys = sorted(inferenceDict.keys())<EOL>for key in keys:<EOL><INDENT>fieldMetaInfo.append(FieldMetaInfo(name=inferenceLabel+"<STR_LIT:.>"+str(key),<EOL>type=FieldMetaType.string,<EOL>special='<STR_LIT>'))<EOL><DEDENT>return fieldMetaInfo<EOL>

Get field metadate information for inferences that are of dict type


<EOL>inferences = modelResult.inferences<EOL>hasInferences = False<EOL>if inferences is not None:<EOL><INDENT>for value in inferences.values():<EOL><INDENT>hasInferences = hasInferences or (value is not None)<EOL><DEDENT><DEDENT>if not hasInferences:<EOL><INDENT>return<EOL><DEDENT>if self.__dataset is None:<EOL><INDENT>self.__openDatafile(modelResult)<EOL><DEDENT>inputData = modelResult.sensorInput<EOL>sequenceReset = int(bool(inputData.sequenceReset))<EOL>outputRow = [sequenceReset]<EOL>rawInput = modelResult.rawInput<EOL>for field in self._rawInputNames:<EOL><INDENT>outputRow.append(str(rawInput[field]))<EOL><DEDENT>for inferenceElement, outputVal in inferences.items():<EOL><INDENT>inputElement = InferenceElement.getInputElement(inferenceElement)<EOL>if inputElement:<EOL><INDENT>inputVal = getattr(inputData, inputElement)<EOL><DEDENT>else:<EOL><INDENT>inputVal = None<EOL><DEDENT>if type(outputVal) in (list, tuple):<EOL><INDENT>assert type(inputVal) in (list, tuple, None)<EOL>for iv, ov in zip(inputVal, outputVal):<EOL><INDENT>outputRow.append(str(iv))<EOL>outputRow.append(str(ov))<EOL><DEDENT><DEDENT>elif isinstance(outputVal, dict):<EOL><INDENT>if inputVal is not None:<EOL><INDENT>if modelResult.predictedFieldName is not None:<EOL><INDENT>outputRow.append(str(inputVal[modelResult.predictedFieldName]))<EOL><DEDENT>else:<EOL><INDENT>outputRow.append(str(inputVal))<EOL><DEDENT><DEDENT>for key in sorted(outputVal.keys()):<EOL><INDENT>outputRow.append(str(outputVal[key]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if inputVal is not None:<EOL><INDENT>outputRow.append(str(inputVal))<EOL><DEDENT>outputRow.append(str(outputVal))<EOL><DEDENT><DEDENT>metrics = modelResult.metrics<EOL>for metricName in self.__metricNames:<EOL><INDENT>outputRow.append(metrics.get(metricName, <NUM_LIT:0.0>))<EOL><DEDENT>self.__dataset.appendRecord(outputRow)<EOL>self.__dataset.flush()<EOL>return<EOL>

[virtual method override] Emits a single prediction as input versus
        predicted.

        modelResult:    An opf_utils.ModelResult object that contains the model input
                        and output for the current timestep.


def checkpoint(self, checkpointSink, maxRows):

checkpointSink.truncate()<EOL>if self.__dataset is None:<EOL><INDENT>if self.__checkpointCache is not None:<EOL><INDENT>self.__checkpointCache.seek(<NUM_LIT:0>)<EOL>shutil.copyfileobj(self.__checkpointCache, checkpointSink)<EOL>checkpointSink.flush()<EOL>return<EOL><DEDENT>else:<EOL><INDENT>return<EOL><DEDENT><DEDENT>self.__dataset.flush()<EOL>totalDataRows = self.__dataset.getDataRowCount()<EOL>if totalDataRows == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>reader = FileRecordStream(self.__datasetPath, missingValues=[])<EOL>writer = csv.writer(checkpointSink)<EOL>writer.writerow(reader.getFieldNames())<EOL>numToWrite = min(maxRows, totalDataRows)<EOL>numRowsToSkip = totalDataRows - numToWrite<EOL>for i in range(numRowsToSkip):<EOL><INDENT>next(reader)<EOL><DEDENT>numWritten = <NUM_LIT:0><EOL>while True:<EOL><INDENT>row = reader.getNextRecord()<EOL>if row is None:<EOL><INDENT>break;<EOL><DEDENT>row =  [str(element) for element in row]<EOL>writer.writerow(row)<EOL>numWritten +=<NUM_LIT:1><EOL><DEDENT>assert numWritten == numToWrite,"<STR_LIT>" % (numWritten, numToWrite)<EOL>checkpointSink.flush()<EOL>return<EOL>

[virtual method override] Save a checkpoint of the prediction output
        stream. The checkpoint comprises up to maxRows of the most recent inference
        records.

        Parameters:
        ----------------------------------------------------------------------
        checkpointSink:     A File-like object where predictions checkpoint data, if
                            any, will be stored.
        maxRows:            Maximum number of most recent inference rows
                            to checkpoint.