def customCompute(self, recordNum, patternNZ, classification):

<EOL>if not hasattr(self, "<STR_LIT>"):<EOL><INDENT>self._computeFlag = False<EOL><DEDENT>if self._computeFlag:<EOL><INDENT>warnings.simplefilter('<STR_LIT:error>', DeprecationWarning)<EOL>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>DeprecationWarning)<EOL><DEDENT>return self._sdrClassifier.compute(recordNum,<EOL>patternNZ,<EOL>classification,<EOL>self.learningMode,<EOL>self.inferenceMode)<EOL>

Just return the inference value from one input sample. The actual
learning happens in compute() -- if, and only if learning is enabled --
which is called when you run the network.

.. warning:: This method is deprecated and exists only to maintain backward
   compatibility. This method is deprecated, and will be removed. Use
   :meth:`nupic.engine.Network.run` instead, which will call
   :meth:`~nupic.regions.sdr_classifier_region.compute`.

:param recordNum: (int) Record number of the input sample.
:param patternNZ: (list) of the active indices from the output below
:param classification: (dict) of the classification information:

       * ``bucketIdx``: index of the encoder bucket
       * ``actValue``:  actual value going into the encoder

:returns: (dict) containing inference results, one entry for each step in
          ``self.steps``. The key is the number of steps, the value is an
          array containing the relative likelihood for each ``bucketIdx``
          starting from 0.

          For example:

          ::

            {'actualValues': [0.0, 1.0, 2.0, 3.0]
              1 : [0.1, 0.3, 0.2, 0.7]
              4 : [0.2, 0.4, 0.3, 0.5]}


return SDRClassifierRegionProto<EOL>

:returns: the pycapnp proto type that the class uses for serialization.


<EOL>self._computeFlag = True<EOL>patternNZ = inputs["<STR_LIT>"].nonzero()[<NUM_LIT:0>]<EOL>if self.learningMode:<EOL><INDENT>categories = [category for category in inputs["<STR_LIT>"]<EOL>if category >= <NUM_LIT:0>]<EOL>if len(categories) > <NUM_LIT:0>:<EOL><INDENT>bucketIdxList = []<EOL>actValueList = []<EOL>for category in categories:<EOL><INDENT>bucketIdxList.append(int(category))<EOL>if "<STR_LIT>" not in inputs:<EOL><INDENT>actValueList.append(int(category))<EOL><DEDENT>else:<EOL><INDENT>actValueList.append(float(inputs["<STR_LIT>"]))<EOL><DEDENT><DEDENT>classificationIn = {"<STR_LIT>": bucketIdxList,<EOL>"<STR_LIT>": actValueList}<EOL><DEDENT>else:<EOL><INDENT>if "<STR_LIT>" not in inputs:<EOL><INDENT>raise KeyError("<STR_LIT>")<EOL><DEDENT>if "<STR_LIT>" not in inputs:<EOL><INDENT>raise KeyError("<STR_LIT>")<EOL><DEDENT>classificationIn = {"<STR_LIT>": int(inputs["<STR_LIT>"]),<EOL>"<STR_LIT>": float(inputs["<STR_LIT>"])}<EOL><DEDENT><DEDENT>else:<EOL><INDENT>classificationIn = {"<STR_LIT>": <NUM_LIT:0>, "<STR_LIT>": <NUM_LIT:0>}<EOL><DEDENT>clResults = self._sdrClassifier.compute(recordNum=self.recordNum,<EOL>patternNZ=patternNZ,<EOL>classification=classificationIn,<EOL>learn=self.learningMode,<EOL>infer=self.inferenceMode)<EOL>if clResults is not None and len(clResults) > <NUM_LIT:0>:<EOL><INDENT>outputs['<STR_LIT>'][:len(clResults["<STR_LIT>"])] =clResults["<STR_LIT>"]<EOL>for step in self.stepsList:<EOL><INDENT>stepIndex = self.stepsList.index(step)<EOL>categoryOut = clResults["<STR_LIT>"][clResults[step].argmax()]<EOL>outputs['<STR_LIT>'][stepIndex] = categoryOut<EOL>stepProbabilities = clResults[step]<EOL>for categoryIndex in xrange(self.maxCategoryCount):<EOL><INDENT>flatIndex = categoryIndex + stepIndex * self.maxCategoryCount<EOL>if categoryIndex < len(stepProbabilities):<EOL><INDENT>outputs['<STR_LIT>'][flatIndex] =stepProbabilities[categoryIndex]<EOL><DEDENT>else:<EOL><INDENT>outputs['<STR_LIT>'][flatIndex] = <NUM_LIT:0.0><EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.recordNum += <NUM_LIT:1><EOL>

Process one input sample.
This method is called by the runtime engine.

:param inputs: (dict) mapping region input names to numpy.array values
:param outputs: (dict) mapping region output names to numpy.arrays that
       should be populated with output values by this method


<EOL>self._normOffset = self._samples.mean(axis=<NUM_LIT:0>) * -<NUM_LIT:1.0><EOL>self._samples += self._normOffset<EOL>variance = self._samples.var(axis=<NUM_LIT:0>)<EOL>variance[numpy.where(variance == <NUM_LIT:0.0>)] = <NUM_LIT:1.0><EOL>self._normScale  = <NUM_LIT:1.0> / numpy.sqrt(variance)<EOL>self._samples *= self._normScale<EOL>for sampleIndex in range(len(self._labels)):<EOL><INDENT>self._knn.learn(self._samples[sampleIndex],<EOL>self._labels[sampleIndex],<EOL>self._partitions[sampleIndex])<EOL><DEDENT>

Compute normalization constants for each feature dimension
based on the collected training samples.  Then normalize our
training samples using these constants (so that each input
dimension has mean and variance of zero and one, respectively.)
Then feed these "sphered" training samples into the underlying
SVM model.


if self._tapFileIn is not None:<EOL><INDENT>for input in inputs:<EOL><INDENT>for k in range(len(input)):<EOL><INDENT>print(input[k], end='<STR_LIT:U+0020>', file=self._tapFileIn)<EOL><DEDENT>print(file=self._tapFileIn)<EOL><DEDENT><DEDENT>

Write inputs to output tap file.

:param inputs: (iter) some inputs.


def getParameter(self, name, index=-<NUM_LIT:1>):

if name == "<STR_LIT>":<EOL><INDENT>return self._knn._numPatterns<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._getPatternMatrix()<EOL><DEDENT>elif name == "<STR_LIT:k>":<EOL><INDENT>return self._knn.k<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.distanceNorm<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.distanceMethod<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.distThreshold<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.binarizationThreshold<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.doBinarization<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.useSparseMemory<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.sparseThreshold<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.numWinners<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.relativeThreshold<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>v = self._knn.numSVDSamples<EOL>return v if v is not None else <NUM_LIT:0><EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>v = self._knn.numSVDDims<EOL>return v if v is not None else <NUM_LIT:0><EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>v = self._knn.fractionOfMax<EOL>return v if v is not None else <NUM_LIT:0><EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._useAuxiliary<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._justUseAuxiliary<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._doSphering<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self._knn.cellsPerCol<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>return self.maxStoredPatterns<EOL><DEDENT>elif name == '<STR_LIT>':<EOL><INDENT>return self._knn._categoryRecencyList<EOL><DEDENT>else:<EOL><INDENT>return PyRegion.getParameter(self, name, index)<EOL><DEDENT>


<EOL>if self._useAuxiliary is None:<EOL><INDENT>self._useAuxiliary = False<EOL><DEDENT>if self._firstComputeCall:<EOL><INDENT>self._firstComputeCall = False<EOL>if self._useAuxiliary:<EOL><INDENT>if self._justUseAuxiliary == True:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>inputVector = inputs['<STR_LIT>']<EOL>if self._useAuxiliary==True:<EOL><INDENT>auxVector = inputs['<STR_LIT>']<EOL>if auxVector.dtype != numpy.float32:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if self._justUseAuxiliary == True:<EOL><INDENT>inputVector = inputs['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>inputVector = numpy.concatenate([inputVector, inputs['<STR_LIT>']])<EOL><DEDENT><DEDENT>self.handleLogInput([inputVector])<EOL>assert "<STR_LIT>" in inputs, "<STR_LIT>"<EOL>categories = inputs['<STR_LIT>']<EOL>if "<STR_LIT>" in inputs:<EOL><INDENT>assert len(inputs["<STR_LIT>"]) == <NUM_LIT:1>, "<STR_LIT>"<EOL>partInput = inputs['<STR_LIT>']<EOL>assert len(partInput) == <NUM_LIT:1>, "<STR_LIT>"<EOL>partition = int(partInput[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>partition = None<EOL><DEDENT>if self.inferenceMode:<EOL><INDENT>categoriesOut = outputs['<STR_LIT>']<EOL>probabilitiesOut = outputs['<STR_LIT>']<EOL>if self._doSphering:<EOL><INDENT>inputVector = (inputVector + self._normOffset) * self._normScale<EOL><DEDENT>nPrototypes = <NUM_LIT:0><EOL>if "<STR_LIT>" in outputs:<EOL><INDENT>bestPrototypeIndicesOut = outputs["<STR_LIT>"]<EOL>nPrototypes = len(bestPrototypeIndicesOut)<EOL><DEDENT>winner, inference, protoScores, categoryDistances =self._knn.infer(inputVector, partitionId=partition)<EOL>if not self.keepAllDistances:<EOL><INDENT>self._protoScores = protoScores<EOL><DEDENT>else:<EOL><INDENT>if self._protoScores is None:<EOL><INDENT>self._protoScores = numpy.zeros((<NUM_LIT:1>, protoScores.shape[<NUM_LIT:0>]),<EOL>protoScores.dtype)<EOL>self._protoScores[<NUM_LIT:0>,:] = protoScores<EOL>self._protoScoreCount = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>if self._protoScoreCount == self._protoScores.shape[<NUM_LIT:0>]:<EOL><INDENT>newProtoScores = numpy.zeros((self._protoScores.shape[<NUM_LIT:0>] * <NUM_LIT:2>,<EOL>self._protoScores.shape[<NUM_LIT:1>]),<EOL>self._protoScores.dtype)<EOL>newProtoScores[:self._protoScores.shape[<NUM_LIT:0>],:] = self._protoScores<EOL>self._protoScores = newProtoScores<EOL><DEDENT>self._protoScores[self._protoScoreCount,:] = protoScores<EOL>self._protoScoreCount += <NUM_LIT:1><EOL><DEDENT><DEDENT>self._categoryDistances = categoryDistances<EOL>if self.outputProbabilitiesByDist:<EOL><INDENT>scores = <NUM_LIT:1.0> - self._categoryDistances<EOL><DEDENT>else:<EOL><INDENT>scores = inference<EOL><DEDENT>total = scores.sum()<EOL>if total == <NUM_LIT:0>:<EOL><INDENT>numScores = len(scores)<EOL>probabilities = numpy.ones(numScores) / numScores<EOL><DEDENT>else:<EOL><INDENT>probabilities = scores / total<EOL><DEDENT>nout = min(len(categoriesOut), len(inference))<EOL>categoriesOut.fill(<NUM_LIT:0>)<EOL>categoriesOut[<NUM_LIT:0>:nout] = inference[<NUM_LIT:0>:nout]<EOL>probabilitiesOut.fill(<NUM_LIT:0>)<EOL>probabilitiesOut[<NUM_LIT:0>:nout] = probabilities[<NUM_LIT:0>:nout]<EOL>if self.verbosity >= <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>", categoriesOut[<NUM_LIT:0>:nout])<EOL>print("<STR_LIT>", probabilitiesOut[<NUM_LIT:0>:nout])<EOL><DEDENT>if self._scanInfo is not None:<EOL><INDENT>self._scanResults = [tuple(inference[:nout])]<EOL><DEDENT>for category in categories:<EOL><INDENT>if category >= <NUM_LIT:0>:<EOL><INDENT>dims = max(int(category)+<NUM_LIT:1>, len(inference))<EOL>oldDims = len(self.confusion)<EOL>if oldDims < dims:<EOL><INDENT>confusion = numpy.zeros((dims, dims))<EOL>confusion[<NUM_LIT:0>:oldDims, <NUM_LIT:0>:oldDims] = self.confusion<EOL>self.confusion = confusion<EOL><DEDENT>self.confusion[inference.argmax(), int(category)] += <NUM_LIT:1><EOL><DEDENT><DEDENT>if nPrototypes > <NUM_LIT:1>:<EOL><INDENT>bestPrototypeIndicesOut.fill(<NUM_LIT:0>)<EOL>if categoryDistances is not None:<EOL><INDENT>indices = categoryDistances.argsort()<EOL>nout = min(len(indices), nPrototypes)<EOL>bestPrototypeIndicesOut[<NUM_LIT:0>:nout] = indices[<NUM_LIT:0>:nout]<EOL><DEDENT><DEDENT>elif nPrototypes == <NUM_LIT:1>:<EOL><INDENT>if (categoryDistances is not None) and len(categoryDistances):<EOL><INDENT>bestPrototypeIndicesOut[<NUM_LIT:0>] = categoryDistances.argmin()<EOL><DEDENT>else:<EOL><INDENT>bestPrototypeIndicesOut[<NUM_LIT:0>] = <NUM_LIT:0><EOL><DEDENT><DEDENT>self.handleLogOutput(inference)<EOL><DEDENT>if self.learningMode:<EOL><INDENT>if (self.acceptanceProbability < <NUM_LIT:1.0>) and(self._rgen.getReal64() > self.acceptanceProbability):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>for category in categories:<EOL><INDENT>if category >= <NUM_LIT:0>:<EOL><INDENT>if self._doSphering:<EOL><INDENT>self._storeSample(inputVector, category, partition)<EOL><DEDENT>else:<EOL><INDENT>self._knn.learn(inputVector, category, partition)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>self._epoch += <NUM_LIT:1><EOL>

Process one input sample. This method is called by the runtime engine.

.. note:: the number of input categories may vary, but the array size is
   fixed to the max number of categories allowed (by a lower region), so
   "unused" indices of the input category array are filled with -1s.

TODO: confusion matrix does not support multi-label classification

:param inputs: (dict) mapping region input names to numpy.array values
:param outputs: (dict) mapping region output names to numpy.arrays that
       should be populated with output values by this method


if self._doSphering:<EOL><INDENT>self._finishSphering()<EOL><DEDENT>self._knn.finishLearning()<EOL>self._accuracy = None<EOL>

Does nothing. Kept here for API compatibility


return self._knn.removeCategory(categoryToRemove)<EOL>

Removes a category.

:param categoryToRemove: (string) label to remove


return self._knn<EOL>


def setParameter(self, name, index, value):

if name == "<STR_LIT>":<EOL><INDENT>self.learningMode = bool(int(value))<EOL>self._epoch = <NUM_LIT:0><EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>self._epoch = <NUM_LIT:0><EOL>if int(value) and not self.inferenceMode:<EOL><INDENT>self._finishLearning()<EOL><DEDENT>self.inferenceMode = bool(int(value))<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>self._knn.distanceNorm = value<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>self._knn.distanceMethod = value<EOL><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>self.keepAllDistances = bool(value)<EOL>if not self.keepAllDistances:<EOL><INDENT>if self._protoScores is not None and self._protoScores.shape[<NUM_LIT:0>] > <NUM_LIT:1>:<EOL><INDENT>self._protoScores = self._protoScores[-<NUM_LIT:1>,:]<EOL><DEDENT>if self._protoScores is not None:<EOL><INDENT>self._protoScoreCount = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self._protoScoreCount = <NUM_LIT:0><EOL><DEDENT><DEDENT><DEDENT>elif name == "<STR_LIT>":<EOL><INDENT>self.verbosity = value<EOL>self._knn.verbosity = value<EOL><DEDENT>else:<EOL><INDENT>return PyRegion.setParameter(self, name, index, value)<EOL><DEDENT>


return ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>

List of attributes to not save with serialized state.


if name == '<STR_LIT>':<EOL><INDENT>return self.maxCategoryCount<EOL><DEDENT>elif name == '<STR_LIT>':<EOL><INDENT>return self.maxCategoryCount<EOL><DEDENT>elif name == '<STR_LIT>':<EOL><INDENT>return self._bestPrototypeIndexCount if self._bestPrototypeIndexCount else <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>' + name)<EOL><DEDENT>


state = self.__dict__.copy()<EOL>state['<STR_LIT>'] = self._knn.__getstate__()<EOL>del state['<STR_LIT>']<EOL>for field in self._getEphemeralAttributes():<EOL><INDENT>del state[field]<EOL><DEDENT>return state<EOL>


self._tapFileIn = open(tapPath + '<STR_LIT>', '<STR_LIT:w>')<EOL>self._tapFileOut = open(tapPath + '<STR_LIT>', '<STR_LIT:w>')<EOL>

Begin writing output tap files.

:param tapPath: (string) base name of the output tap files to write.


<EOL>if self._tapFileOut is not None:<EOL><INDENT>for k in range(len(output)):<EOL><INDENT>print(output[k], end='<STR_LIT:U+0020>', file=self._tapFileOut)<EOL><DEDENT>print(file=self._tapFileOut)<EOL><DEDENT>

Write outputs to output tap file.

:param outputs: (iter) some outputs.


if '<STR_LIT:version>' not in state:<EOL><INDENT>self.__dict__.update(state)<EOL><DEDENT>elif state['<STR_LIT:version>'] == <NUM_LIT:1>:<EOL><INDENT>if "<STR_LIT>" in state:<EOL><INDENT>state.pop("<STR_LIT>")<EOL><DEDENT>knnState = state['<STR_LIT>']<EOL>del state['<STR_LIT>']<EOL>self.__dict__.update(state)<EOL>self._initEphemerals()<EOL>self._knn.__setstate__(knnState)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>self.version = KNNClassifierRegion.__VERSION__<EOL>


<EOL>assert isinstance(self.description, str)<EOL>assert isinstance(self.singleNodeOnly, bool)<EOL>assert isinstance(self.inputs, dict)<EOL>assert isinstance(self.outputs, dict)<EOL>assert isinstance(self.parameters, dict)<EOL>assert isinstance(self.commands, dict)<EOL>hasDefaultInput = False<EOL>for k, v in self.inputs.items():<EOL><INDENT>assert isinstance(k, str)<EOL>assert isinstance(v, InputSpec)<EOL>v.invariant()<EOL>if v.isDefaultInput:<EOL><INDENT>assert not hasDefaultInput<EOL>hasDefaultInput = True<EOL><DEDENT><DEDENT>hasDefaultOutput = False<EOL>for k, v in self.outputs.items():<EOL><INDENT>assert isinstance(k, str)<EOL>assert isinstance(v, OutputSpec)<EOL>v.invariant()<EOL>if v.isDefaultOutput:<EOL><INDENT>assert not hasDefaultOutput<EOL>hasDefaultOutput = True<EOL><DEDENT><DEDENT>for k, v in self.parameters.items():<EOL><INDENT>assert isinstance(k, str)<EOL>assert isinstance(v, ParameterSpec)<EOL>v.invariant()<EOL><DEDENT>for k, v in self.commands.items():<EOL><INDENT>assert isinstance(k, str)<EOL>assert isinstance(v, CommandSpec)<EOL>v.invariant()<EOL><DEDENT>

Verify the validity of the node spec object

        The type of each sub-object is verified and then
        the validity of each node spec item is verified by calling
        it invariant() method. It also makes sure that there is at most
        one default input and one default output.


return '<STR_LIT>'<EOL>

Return the default spatial pooler implementation for this region.


def _doTopDownInfer(self, topDownInput = None):

return None, None<EOL>

Do one iteration of top-down inference.

Parameters:
--------------------------------------------
tdInput:      Top-down input

retval:     (spatialTopDownOut, temporalTopDownOut)
              spatialTopDownOut is the top down output computed only from the SP,
                using it's current bottom-up output.
              temporalTopDownOut is the top down output computed from the topDown in
               of the level above us.


return self._getEphemeralMembersBase() + self._getEphemeralMembers()<EOL>

Returns a concatenated list of both the standard base class
ephemeral members, as well as any additional ephemeral members
(e.g., file handles, etc.).


Doesn't include the spatial, temporal and other parameters

:returns: (dict) The base Spec for SPRegion.


if spatialImp == '<STR_LIT>':<EOL><INDENT>return PYSpatialPooler<EOL><DEDENT>elif spatialImp == '<STR_LIT>':<EOL><INDENT>return CPPSpatialPooler<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (spatialImp))<EOL><DEDENT>

Return the class corresponding to the given spatialImp string


<EOL>self._spatialPoolerOutput = numpy.zeros(self.columnCount,<EOL>dtype=GetNTAReal())<EOL>self._spatialPoolerInput = numpy.zeros((<NUM_LIT:1>, self.inputWidth),<EOL>dtype=GetNTAReal())<EOL>self._allocateSpatialFDR(None)<EOL>


<EOL>if False and self.learningModeand self._iterations > <NUM_LIT:0> and self._iterations <= <NUM_LIT:10>:<EOL><INDENT>import hotshot<EOL>if self._iterations == <NUM_LIT:10>:<EOL><INDENT>print("<STR_LIT>")<EOL>stats = hotshot.stats.load("<STR_LIT>")<EOL>stats.strip_dirs()<EOL>stats.sort_stats('<STR_LIT:time>', '<STR_LIT>')<EOL>stats.print_stats()<EOL><DEDENT>if self._profileObj is None:<EOL><INDENT>print("<STR_LIT>")<EOL>if os.path.exists('<STR_LIT>'):<EOL><INDENT>os.remove('<STR_LIT>')<EOL><DEDENT>self._profileObj = hotshot.Profile("<STR_LIT>", <NUM_LIT:1>, <NUM_LIT:1>)<EOL><DEDENT>self._profileObj.runcall(self._compute, *[inputs, outputs])<EOL><DEDENT>else:<EOL><INDENT>self._compute(inputs, outputs)<EOL><DEDENT>

Run one iteration, profiling it if requested.

:param inputs: (dict) mapping region input names to numpy.array values
:param outputs: (dict) mapping region output names to numpy.arrays that
       should be populated with output values by this method


def _doBottomUpCompute(self, rfInput, resetSignal):

<EOL>self._conditionalBreak()<EOL>self._spatialPoolerInput = rfInput.reshape(-<NUM_LIT:1>)<EOL>assert(rfInput.shape[<NUM_LIT:0>] == <NUM_LIT:1>)<EOL>inputVector = numpy.array(rfInput[<NUM_LIT:0>]).astype('<STR_LIT>')<EOL>outputVector = numpy.zeros(self._sfdr.getNumColumns()).astype('<STR_LIT>')<EOL>self._sfdr.compute(inputVector, self.learningMode, outputVector)<EOL>self._spatialPoolerOutput[:] = outputVector[:]<EOL>if self._fpLogSP:<EOL><INDENT>output = self._spatialPoolerOutput.reshape(-<NUM_LIT:1>)<EOL>outputNZ = output.nonzero()[<NUM_LIT:0>]<EOL>outStr = "<STR_LIT:U+0020>".join(["<STR_LIT>" % int(token) for token in outputNZ])<EOL>print(output.size, outStr, file=self._fpLogSP)<EOL><DEDENT>if self._fpLogSPInput:<EOL><INDENT>output = rfInput.reshape(-<NUM_LIT:1>)<EOL>outputNZ = output.nonzero()[<NUM_LIT:0>]<EOL>outStr = "<STR_LIT:U+0020>".join(["<STR_LIT>" % int(token) for token in outputNZ])<EOL>print(output.size, outStr, file=self._fpLogSPInput)<EOL><DEDENT>return self._spatialPoolerOutput<EOL>

Do one iteration of inference and/or learning and return the result

Parameters:
--------------------------------------------
rfInput:      Input vector. Shape is: (1, inputVectorLen).
resetSignal:  True if reset is asserted


typeNames = {int: '<STR_LIT>', float: '<STR_LIT>', str: '<STR_LIT>', bool: '<STR_LIT:bool>', tuple: '<STR_LIT>'}<EOL>def getArgType(arg):<EOL><INDENT>t = typeNames.get(type(arg), '<STR_LIT>')<EOL>count = <NUM_LIT:0> if t == '<STR_LIT>' else <NUM_LIT:1><EOL>if t == '<STR_LIT>':<EOL><INDENT>t = typeNames.get(type(arg[<NUM_LIT:0>]), '<STR_LIT>')<EOL>count = len(arg)<EOL><DEDENT>if t == '<STR_LIT:bool>':<EOL><INDENT>t = '<STR_LIT>'<EOL><DEDENT>return (t, count)<EOL><DEDENT>def getConstraints(arg):<EOL><INDENT>t = typeNames.get(type(arg), '<STR_LIT>')<EOL>if t == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>elif t == '<STR_LIT:bool>':<EOL><INDENT>return '<STR_LIT:bool>'<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT><DEDENT>SpatialClass = getSPClass(spatialImp)<EOL>sArgTuples = _buildArgs(SpatialClass.__init__)<EOL>spatialSpec = {}<EOL>for argTuple in sArgTuples:<EOL><INDENT>d = dict(<EOL>description=argTuple[<NUM_LIT:1>],<EOL>accessMode='<STR_LIT>',<EOL>dataType=getArgType(argTuple[<NUM_LIT:2>])[<NUM_LIT:0>],<EOL>count=getArgType(argTuple[<NUM_LIT:2>])[<NUM_LIT:1>],<EOL>constraints=getConstraints(argTuple[<NUM_LIT:2>]))<EOL>spatialSpec[argTuple[<NUM_LIT:0>]] = d<EOL><DEDENT>spatialSpec.update(dict(<EOL>columnCount=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>inputWidth=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>spInputNonZeros=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>spOutputNonZeros=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>spOverlapDistribution=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>sparseCoincidenceMatrix=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>denseOutput=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>spLearningStatsStr=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>spatialImp=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>))<EOL>otherSpec = dict(<EOL>learningMode=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT:bool>'),<EOL>inferenceMode=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT:bool>'),<EOL>anomalyMode=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT:bool>'),<EOL>topDownMode=dict(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT:bool>'),<EOL>activeOutputCount=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>logPathInput=dict(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>logPathOutput=dict(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>logPathOutputDense=dict(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>)<EOL>return spatialSpec, otherSpec<EOL>

Build the additional specs in three groups (for the inspector)

    Use the type of the default argument to set the Spec type, defaulting
    to 'Byte' for None and complex types

    Determines the spatial parameters based on the selected implementation.
    It defaults to SpatialPooler.


self.__dict__.update(state)<EOL>self._loaded = True<EOL>if not hasattr(self, "<STR_LIT>"):<EOL><INDENT>self.SpatialClass = self._sfdr.__class__<EOL><DEDENT>self._initializeEphemeralMembers()<EOL>self._allocateSpatialFDR(None)<EOL>

Set the state of ourself from a serialized state.


return [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>]<EOL>


if label not in self.saved_categories:<EOL><INDENT>self.saved_categories.append(label)<EOL><DEDENT>return pow(<NUM_LIT:2>, self.saved_categories.index(label))<EOL>

Since the KNN Classifier stores categories as numbers, we must store each
label as a number. This method converts from a label to a unique number.
Each label is assigned a unique bit so multiple labels may be assigned to
a single record.


def getLabels(self, start=None, end=None):

if len(self._recordsCache) == <NUM_LIT:0>:<EOL><INDENT>return {<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': []<EOL>}<EOL><DEDENT>try:<EOL><INDENT>start = int(start)<EOL><DEDENT>except Exception:<EOL><INDENT>start = <NUM_LIT:0><EOL><DEDENT>try:<EOL><INDENT>end = int(end)<EOL><DEDENT>except Exception:<EOL><INDENT>end = self._recordsCache[-<NUM_LIT:1>].ROWID<EOL><DEDENT>if end <= start:<EOL><INDENT>raise HTMPredictionModelInvalidRangeError("<STR_LIT>",<EOL>debugInfo={<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': start,<EOL>'<STR_LIT>': end<EOL>},<EOL>'<STR_LIT>': len(self._recordsCache)<EOL>})<EOL><DEDENT>results = {<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': []<EOL>}<EOL>ROWIDX = numpy.array(<EOL>self._knnclassifier.getParameter('<STR_LIT>'))<EOL>validIdx = numpy.where((ROWIDX >= start) & (ROWIDX < end))[<NUM_LIT:0>].tolist()<EOL>categories = self._knnclassifier.getCategoryList()<EOL>for idx in validIdx:<EOL><INDENT>row = dict(<EOL>ROWID=int(ROWIDX[idx]),<EOL>labels=self._categoryToLabelList(categories[idx]))<EOL>results['<STR_LIT>'].append(row)<EOL><DEDENT>return results<EOL>

Get the labels on classified points within range start to end. Not inclusive
of end.

:returns: (dict) with format:

  ::

    {
      'isProcessing': boolean,
      'recordLabels': list of results
    }

  ``isProcessing`` - currently always false as recalculation blocks; used if
  reprocessing of records is still being performed;

  Each item in ``recordLabels`` is of format:

  ::

    {
      'ROWID': id of the row,
      'labels': list of strings
    }


return self.labelResults<EOL>

Get the labels of the previously computed record.

:returns: (list) of strings representing the classification labels


record = self._constructClassificationRecord(inputs)<EOL>if record.ROWID >= self.getParameter('<STR_LIT>'):<EOL><INDENT>self._classifyState(record)<EOL><DEDENT>self._recordsCache.append(record)<EOL>while len(self._recordsCache) > self.cacheSize:<EOL><INDENT>self._recordsCache.pop(<NUM_LIT:0>)<EOL><DEDENT>self.labelResults = record.anomalyLabel<EOL>self._iteration += <NUM_LIT:1><EOL>

Process one input sample.
This method is called by the runtime engine.


if name == '<STR_LIT>':<EOL><INDENT>return self._maxLabelOutputs<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>


def _deleteRangeFromKNN(self, start=<NUM_LIT:0>, end=None):

prototype_idx = numpy.array(<EOL>self._knnclassifier.getParameter('<STR_LIT>'))<EOL>if end is None:<EOL><INDENT>end = prototype_idx.max() + <NUM_LIT:1><EOL><DEDENT>idsIdxToDelete = numpy.logical_and(prototype_idx >= start,<EOL>prototype_idx < end)<EOL>idsToDelete = prototype_idx[idsIdxToDelete]<EOL>nProtos = self._knnclassifier._knn._numPatterns<EOL>self._knnclassifier._knn.removeIds(idsToDelete.tolist())<EOL>assert self._knnclassifier._knn._numPatterns == nProtos - len(idsToDelete)<EOL>

Removes any stored records within the range from start to
end. Noninclusive of end.

parameters
------------
start - integer representing the ROWID of the start of the deletion range,
end - integer representing the ROWID of the end of the deletion range,
  if None, it will default to end.


def addLabel(self, start, end, labelName):

if len(self._recordsCache) == <NUM_LIT:0>:<EOL><INDENT>raise HTMPredictionModelInvalidRangeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>start = int(start)<EOL><DEDENT>except Exception:<EOL><INDENT>start = <NUM_LIT:0><EOL><DEDENT>try:<EOL><INDENT>end = int(end)<EOL><DEDENT>except Exception:<EOL><INDENT>end = int(self._recordsCache[-<NUM_LIT:1>].ROWID)<EOL><DEDENT>startID = self._recordsCache[<NUM_LIT:0>].ROWID<EOL>clippedStart = max(<NUM_LIT:0>, start - startID)<EOL>clippedEnd = max(<NUM_LIT:0>, min( len( self._recordsCache) , end - startID))<EOL>if clippedEnd <= clippedStart:<EOL><INDENT>raise HTMPredictionModelInvalidRangeError("<STR_LIT>",<EOL>debugInfo={<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': start,<EOL>'<STR_LIT>': end<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': clippedStart,<EOL>'<STR_LIT>': clippedEnd<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': startID,<EOL>'<STR_LIT>': self._recordsCache[len(self._recordsCache)-<NUM_LIT:1>].ROWID<EOL>},<EOL>'<STR_LIT>': len(self._recordsCache)<EOL>})<EOL><DEDENT>for state in self._recordsCache[clippedStart:clippedEnd]:<EOL><INDENT>if labelName not in state.anomalyLabel:<EOL><INDENT>state.anomalyLabel.append(labelName)<EOL>state.setByUser = True<EOL>self._addRecordToKNN(state)<EOL><DEDENT><DEDENT>assert len(self.saved_categories) > <NUM_LIT:0><EOL>for state in self._recordsCache[clippedEnd:]:<EOL><INDENT>self._classifyState(state)<EOL><DEDENT>

Add the label labelName to each record with record ROWID in range from
``start`` to ``end``, noninclusive of end.

This will recalculate all points from end to the last record stored in the
internal cache of this classifier.

:param start: (int) start index
:param end: (int) end index (noninclusive)
:param labelName: (string) label name


<EOL>if state.ROWID < self.getParameter('<STR_LIT>'):<EOL><INDENT>if not state.setByUser:<EOL><INDENT>state.anomalyLabel = []<EOL>self._deleteRecordsFromKNN([state])<EOL><DEDENT>return<EOL><DEDENT>label = KNNAnomalyClassifierRegion.AUTO_THRESHOLD_CLASSIFIED_LABEL<EOL>autoLabel = label + KNNAnomalyClassifierRegion.AUTO_TAG<EOL>newCategory = self._recomputeRecordFromKNN(state)<EOL>labelList = self._categoryToLabelList(newCategory)<EOL>if state.setByUser:<EOL><INDENT>if label in state.anomalyLabel:<EOL><INDENT>state.anomalyLabel.remove(label)<EOL><DEDENT>if autoLabel in state.anomalyLabel:<EOL><INDENT>state.anomalyLabel.remove(autoLabel)<EOL><DEDENT>labelList.extend(state.anomalyLabel)<EOL><DEDENT>if state.anomalyScore >= self.getParameter('<STR_LIT>'):<EOL><INDENT>labelList.append(label)<EOL><DEDENT>elif label in labelList:<EOL><INDENT>ind = labelList.index(label)<EOL>labelList[ind] = autoLabel<EOL><DEDENT>labelList = list(set(labelList))<EOL>if label in labelList and autoLabel in labelList:<EOL><INDENT>labelList.remove(autoLabel)<EOL><DEDENT>if state.anomalyLabel == labelList:<EOL><INDENT>return<EOL><DEDENT>state.anomalyLabel = labelList<EOL>if state.anomalyLabel == []:<EOL><INDENT>self._deleteRecordsFromKNN([state])<EOL><DEDENT>else:<EOL><INDENT>self._addRecordToKNN(state)<EOL><DEDENT>


inputs = {<EOL>"<STR_LIT>": [None],<EOL>"<STR_LIT>": self._getStateAnomalyVector(record),<EOL>}<EOL>outputs = {"<STR_LIT>": numpy.zeros((<NUM_LIT:1>,)),<EOL>"<STR_LIT>":numpy.zeros((<NUM_LIT:1>,)),<EOL>"<STR_LIT>":numpy.zeros((<NUM_LIT:1>,))}<EOL>classifier_indexes = numpy.array(<EOL>self._knnclassifier.getParameter('<STR_LIT>'))<EOL>valid_idx = numpy.where(<EOL>(classifier_indexes >= self.getParameter('<STR_LIT>')) &<EOL>(classifier_indexes < record.ROWID)<EOL>)[<NUM_LIT:0>].tolist()<EOL>if len(valid_idx) == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>self._knnclassifier.setParameter('<STR_LIT>', None, True)<EOL>self._knnclassifier.setParameter('<STR_LIT>', None, False)<EOL>self._knnclassifier.compute(inputs, outputs)<EOL>self._knnclassifier.setParameter('<STR_LIT>', None, True)<EOL>classifier_distances = self._knnclassifier.getLatestDistances()<EOL>valid_distances = classifier_distances[valid_idx]<EOL>if valid_distances.min() <= self._classificationMaxDist:<EOL><INDENT>classifier_indexes_prev = classifier_indexes[valid_idx]<EOL>rowID = classifier_indexes_prev[valid_distances.argmin()]<EOL>indexID = numpy.where(classifier_indexes == rowID)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>category = self._knnclassifier.getCategoryList()[indexID]<EOL>return category<EOL><DEDENT>return None<EOL>

returns the classified labeling of record


knn = self._knnclassifier._knn<EOL>prototype_idx = self._knnclassifier.getParameter('<STR_LIT>')<EOL>category = self._labelListToCategoryNumber(record.anomalyLabel)<EOL>if record.ROWID in prototype_idx:<EOL><INDENT>knn.prototypeSetCategory(record.ROWID, category)<EOL>return<EOL><DEDENT>pattern = self._getStateAnomalyVector(record)<EOL>rowID = record.ROWID<EOL>knn.learn(pattern, category, rowID=rowID)<EOL>


<EOL>allSPColumns = inputs["<STR_LIT>"]<EOL>activeSPColumns = allSPColumns.nonzero()[<NUM_LIT:0>]<EOL>score = anomaly.computeRawAnomalyScore(activeSPColumns,<EOL>self._prevPredictedColumns)<EOL>spSize = len(allSPColumns)<EOL>allTPCells = inputs['<STR_LIT>']<EOL>tpSize = len(inputs['<STR_LIT>'])<EOL>classificationVector = numpy.array([])<EOL>if self.classificationVectorType == <NUM_LIT:1>:<EOL><INDENT>classificationVector = numpy.zeros(tpSize)<EOL>activeCellMatrix = inputs["<STR_LIT>"].reshape(tpSize, <NUM_LIT:1>)<EOL>activeCellIdx = numpy.where(activeCellMatrix > <NUM_LIT:0>)[<NUM_LIT:0>]<EOL>if activeCellIdx.shape[<NUM_LIT:0>] > <NUM_LIT:0>:<EOL><INDENT>classificationVector[numpy.array(activeCellIdx, dtype=numpy.uint16)] = <NUM_LIT:1><EOL><DEDENT><DEDENT>elif self.classificationVectorType == <NUM_LIT:2>:<EOL><INDENT>classificationVector = numpy.zeros(spSize+spSize)<EOL>if activeSPColumns.shape[<NUM_LIT:0>] > <NUM_LIT:0>:<EOL><INDENT>classificationVector[activeSPColumns] = <NUM_LIT:1.0><EOL><DEDENT>errorColumns = numpy.setdiff1d(self._prevPredictedColumns,<EOL>activeSPColumns)<EOL>if errorColumns.shape[<NUM_LIT:0>] > <NUM_LIT:0>:<EOL><INDENT>errorColumnIndexes = ( numpy.array(errorColumns, dtype=numpy.uint16) +<EOL>spSize )<EOL>classificationVector[errorColumnIndexes] = <NUM_LIT:1.0><EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>" % (self.classificationVectorType))<EOL><DEDENT>numPredictedCols = len(self._prevPredictedColumns)<EOL>predictedColumns = allTPCells.nonzero()[<NUM_LIT:0>]<EOL>self._prevPredictedColumns = copy.deepcopy(predictedColumns)<EOL>if self._anomalyVectorLength is None:<EOL><INDENT>self._anomalyVectorLength = len(classificationVector)<EOL><DEDENT>result = _CLAClassificationRecord(<EOL>ROWID=self._iteration, <EOL>anomalyScore=score,<EOL>anomalyVector=classificationVector.nonzero()[<NUM_LIT:0>].tolist(),<EOL>anomalyLabel=[]<EOL>)<EOL>return result<EOL>

Construct a _HTMClassificationRecord based on the state of the model
passed in through the inputs.

Types for self.classificationVectorType:
  1 - TM active cells in learn state
  2 - SP columns concatenated with error from TM column predictions and SP


vector = numpy.zeros(self._anomalyVectorLength)<EOL>vector[state.anomalyVector] = <NUM_LIT:1><EOL>return vector<EOL>

Returns a state's anomaly vertor converting it from spare to dense


state = self.__dict__.copy()<EOL>state['<STR_LIT>'] = state['<STR_LIT>'].__getstate__()<EOL>state.pop('<STR_LIT>')<EOL>return state<EOL>

Return serializable state.  This function will return a version of the
__dict__ with all "ephemeral" members stripped out.  "Ephemeral" members
are defined as those that do not need to be (nor should be) stored
in any kind of persistent file (e.g., NuPIC network XML file.)


assert not self.identityPolicy<EOL>assert isinstance(identityPolicyObj, RegionIdentityPolicyBase)<EOL>self.identityPolicy = identityPolicyObj<EOL>return<EOL>

TestRegion command that sets identity policy instance.  The instance
        MUST be derived from TestRegion's RegionIdentityPolicyBase class.

        Users MUST set the identity instance BEFORE running the network

        Exception: AssertionError if identity policy instance has already been set
                   or if the passed-in instance is not derived from
                   RegionIdentityPolicyBase.


self.identityPolicy.initialize(self)<EOL>_debugOut(self.identityPolicy.getName())<EOL>

Called by network after all links have been set up


state = self.__dict__.copy()<EOL>state.pop('<STR_LIT>')<EOL>return state<EOL>


@abstractmethod<EOL><INDENT>def compute(self, inputs, outputs):<DEDENT>

Perform the main computation

        This method is called in each iteration for each phase the node supports.

        Called from the scope of the region's PyRegion.compute() method.

        inputs: dict of numpy arrays (one per input)
        outputs: dict of numpy arrays (one per output)


assert '<STR_LIT>' not in state<EOL>self.__dict__.update(state)<EOL>self.__constructEphemeralInstanceVars()<EOL>return<EOL>


import inspect<EOL>frameObj = inspect.stack()[<NUM_LIT:2>][<NUM_LIT:0>]<EOL>return inspect.getframeinfo(frameObj)<EOL>

Returns: Traceback namedtuple for our caller's caller


def getParameter(self, parameterName, index=-<NUM_LIT:1>):

assert not (parameterName in self.__dict__ and parameterName in self.ephemeral)<EOL>if parameterName in self.ephemeral:<EOL><INDENT>assert parameterName not in self.__dict__<EOL>return self.ephemeral[parameterName]<EOL><DEDENT>else:<EOL><INDENT>return super(PyRegion, self).getParameter(parameterName, index)<EOL><DEDENT>

Get the value of a NodeSpec parameter. Most parameters are handled
automatically by PyRegion's parameter get mechanism. The ones that need
special treatment are explicitly handled here.


<EOL>self.identityPolicy.compute(inputs, outputs)<EOL>_debugOut(("<STR_LIT>") %(self.identityPolicy.getName(),inputs, outputs))<EOL>return<EOL>

Run one iteration of the region's compute.

The guts of the compute are contained in the _compute() call so that
we can profile it if requested.


Return the number of elements in the given output of the region

        Called from the scope of the region's PyRegion.getOutputElementCount() method.

        name: the name of the output


return RecordSensorProto<EOL>


if not self.topDownMode:<EOL><INDENT>data = self.getNextRecord()<EOL>reset = data["<STR_LIT>"]<EOL>sequenceId = data["<STR_LIT>"]<EOL>categories = data["<STR_LIT>"]<EOL>self.encoder.encodeIntoArray(data, outputs["<STR_LIT>"])<EOL>if self.predictedField is not None and self.predictedField != "<STR_LIT>":<EOL><INDENT>allEncoders = list(self.encoder.encoders)<EOL>if self.disabledEncoder is not None:<EOL><INDENT>allEncoders.extend(self.disabledEncoder.encoders)<EOL><DEDENT>encoders = [e for e in allEncoders<EOL>if e[<NUM_LIT:0>] == self.predictedField]<EOL>if len(encoders) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>" % self.predictedField)<EOL><DEDENT>else:<EOL><INDENT>encoder = encoders[<NUM_LIT:0>][<NUM_LIT:1>]<EOL><DEDENT>actualValue = data[self.predictedField]<EOL>outputs["<STR_LIT>"][:] = encoder.getBucketIndices(actualValue)<EOL>if isinstance(actualValue, str):<EOL><INDENT>outputs["<STR_LIT>"][:] = encoder.getBucketIndices(actualValue)<EOL><DEDENT>else:<EOL><INDENT>outputs["<STR_LIT>"][:] = actualValue<EOL><DEDENT><DEDENT>outputs["<STR_LIT>"][:] = self.encoder.getScalars(data)<EOL>self._outputValues["<STR_LIT>"] = self.encoder.getEncodedValues(data)<EOL>encoders = self.encoder.getEncoderList()<EOL>prevOffset = <NUM_LIT:0><EOL>sourceEncodings = []<EOL>bitData = outputs["<STR_LIT>"]<EOL>for encoder in encoders:<EOL><INDENT>nextOffset = prevOffset + encoder.getWidth()<EOL>sourceEncodings.append(bitData[prevOffset:nextOffset])<EOL>prevOffset = nextOffset<EOL><DEDENT>self._outputValues['<STR_LIT>'] = sourceEncodings<EOL>for filter in self.postEncodingFilters:<EOL><INDENT>filter.process(encoder=self.encoder, data=outputs['<STR_LIT>'])<EOL><DEDENT>outputs['<STR_LIT>'][<NUM_LIT:0>] = reset<EOL>outputs['<STR_LIT>'][<NUM_LIT:0>] = sequenceId<EOL>self.populateCategoriesOut(categories, outputs['<STR_LIT>'])<EOL>if self.verbosity >= <NUM_LIT:1>:<EOL><INDENT>if self._iterNum == <NUM_LIT:0>:<EOL><INDENT>self.encoder.pprintHeader(prefix="<STR_LIT>")<EOL><DEDENT>if reset:<EOL><INDENT>print("<STR_LIT>" % sequenceId)<EOL><DEDENT>if self.verbosity >= <NUM_LIT:2>:<EOL><INDENT>print()<EOL><DEDENT><DEDENT>if self.verbosity >= <NUM_LIT:1>:<EOL><INDENT>self.encoder.pprint(outputs["<STR_LIT>"], prefix="<STR_LIT>" % (self._iterNum))<EOL>scalarValues = self.encoder.getScalars(data)<EOL>nz = outputs["<STR_LIT>"].nonzero()[<NUM_LIT:0>]<EOL>print("<STR_LIT>" % (len(nz)), nz)<EOL>print("<STR_LIT>", self.encoder.scalarsToStr(scalarValues))<EOL><DEDENT>if self.verbosity >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>", str(data))<EOL><DEDENT>if self.verbosity >= <NUM_LIT:3>:<EOL><INDENT>decoded = self.encoder.decode(outputs["<STR_LIT>"])<EOL>print("<STR_LIT>", self.encoder.decodedToStr(decoded))<EOL><DEDENT>self._iterNum += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>spatialTopDownIn = inputs['<STR_LIT>']<EOL>spatialTopDownOut = self.encoder.topDownCompute(spatialTopDownIn)<EOL>values = [elem.value for elem in spatialTopDownOut]<EOL>scalars = [elem.scalar for elem in spatialTopDownOut]<EOL>encodings = [elem.encoding for elem in spatialTopDownOut]<EOL>self._outputValues['<STR_LIT>'] = values<EOL>outputs['<STR_LIT>'][:] = numpy.array(scalars)<EOL>self._outputValues['<STR_LIT>'] = encodings<EOL>temporalTopDownIn = inputs['<STR_LIT>']<EOL>temporalTopDownOut = self.encoder.topDownCompute(temporalTopDownIn)<EOL>values = [elem.value for elem in temporalTopDownOut]<EOL>scalars = [elem.scalar for elem in temporalTopDownOut]<EOL>encodings = [elem.encoding for elem in temporalTopDownOut]<EOL>self._outputValues['<STR_LIT>'] = values<EOL>outputs['<STR_LIT>'][:] = numpy.array(scalars)<EOL>self._outputValues['<STR_LIT>'] = encodings<EOL>assert len(spatialTopDownOut) == len(temporalTopDownOut), (<EOL>"<STR_LIT>"<EOL>"<STR_LIT:size>")<EOL><DEDENT>

Get a record from the dataSource and encode it.

Overrides :meth:`nupic.bindings.regions.PyRegion.PyRegion.compute`.


def setParameter(self, parameterName, index, parameterValue):

if parameterName == '<STR_LIT>':<EOL><INDENT>self.topDownMode = parameterValue<EOL><DEDENT>elif parameterName == '<STR_LIT>':<EOL><INDENT>self.predictedField = parameterValue<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>' + parameterName)<EOL><DEDENT>

Set the value of a Spec parameter. Most parameters are handled
automatically by PyRegion's parameter set mechanism. The ones that need
special treatment are explicitly handled here.


def populateCategoriesOut(self, categories, output):

if categories[<NUM_LIT:0>] is None:<EOL><INDENT>output[:] = -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>for i, cat in enumerate(categories[:len(output)]):<EOL><INDENT>output[i] = cat<EOL><DEDENT>output[len(categories):] = -<NUM_LIT:1><EOL><DEDENT>

Populate the output array with the category indices.

.. note:: Non-categories are represented with ``-1``.

:param categories: (list) of category strings
:param output: (list) category output, will be overwritten


instance = cls()<EOL>instance.encoder = MultiEncoder.read(proto.encoder)<EOL>if proto.disabledEncoder is not None:<EOL><INDENT>instance.disabledEncoder = MultiEncoder.read(proto.disabledEncoder)<EOL><DEDENT>instance.topDownMode = bool(proto.topDownMode)<EOL>instance.verbosity = proto.verbosity<EOL>instance.numCategories = proto.numCategories<EOL>return instance<EOL>


self._sensedValue = value<EOL>

:param value: will be encoded when this region does a compute.


return self._sensedValue<EOL>


def getLabel(cls, val):<EOL><INDENT>"""<STR_LIT>"""<EOL>return cls.__labels[val]<EOL><DEDENT>def validate(cls, val):<EOL><INDENT>"""<STR_LIT>"""<EOL>return val in cls.__values<EOL><DEDENT>def getValues(cls):<EOL><INDENT>"""<STR_LIT>"""<EOL>return list(cls.__values)<EOL><DEDENT>def getLabels(cls):<EOL><INDENT>"""<STR_LIT>"""<EOL>return list(cls.__labels.values())<EOL><DEDENT>def getValue(cls, label):<EOL><INDENT>"""<STR_LIT>"""<EOL>return cls.__labels[label]<EOL><DEDENT>for arg in list(args)+kwargs.keys():<EOL><INDENT>if type(arg) is not str:<EOL><INDENT>raise TypeError("<STR_LIT>".format(arg))<EOL><DEDENT>if not __isidentifier(arg):<EOL><INDENT>raise ValueError("<STR_LIT>""<STR_LIT>".format(arg))<EOL><DEDENT><DEDENT>kwargs.update(zip(args, args))<EOL>newType = type("<STR_LIT>", (object,), kwargs)<EOL>newType.__labels = dict( (v,k) for k,v in kwargs.iteritems())<EOL>newType.__values = set(newType.__labels.keys())<EOL>newType.getLabel = functools.partial(getLabel, newType)<EOL>newType.validate = functools.partial(validate, newType)<EOL>newType.getValues = functools.partial(getValues, newType)<EOL>newType.getLabels = functools.partial(getLabels, newType)<EOL>newType.getValue = functools.partial(getValue, newType)<EOL>return newType<EOL>

Utility function for creating enumerations in python

Example Usage:
  >> Color = Enum("Red", "Green", "Blue", "Magenta")
  >> print Color.Red
  >> 0
  >> print Color.Green
  >> 1
  >> print Color.Blue
  >> 2
  >> print Color.Magenta
  >> 3
  >> Color.Violet
  >> 'violet'
  >> Color.getLabel(Color.Red)
  >> 'Red'
  >> Color.getLabel(2)
  >> 'Blue'


generatorList = [] <EOL>if len(args) % <NUM_LIT:2> == <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>advanceList = []<EOL>for i in xrange(<NUM_LIT:0>, len(args), <NUM_LIT:2>):<EOL><INDENT>listn = args[i]<EOL>fn = args[i + <NUM_LIT:1>]<EOL>if listn is not None:<EOL><INDENT>generatorList.append(groupby(listn, fn))<EOL>advanceList.append(True) <EOL><DEDENT>else:<EOL><INDENT>generatorList.append(None)<EOL>advanceList.append(False)<EOL><DEDENT><DEDENT>n = len(generatorList)<EOL>nextList = [None] * n<EOL>while True:<EOL><INDENT>for i in xrange(n):<EOL><INDENT>if advanceList[i]:<EOL><INDENT>try:<EOL><INDENT>nextList[i] = generatorList[i].next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>nextList[i] = None<EOL><DEDENT><DEDENT><DEDENT>if all(entry is None for entry in nextList):<EOL><INDENT>break<EOL><DEDENT>minKeyVal = min(nextVal[<NUM_LIT:0>] for nextVal in nextList<EOL>if nextVal is not None)<EOL>retGroups = [minKeyVal]<EOL>for i in xrange(n):<EOL><INDENT>if nextList[i] is not None and nextList[i][<NUM_LIT:0>] == minKeyVal:<EOL><INDENT>retGroups.append(nextList[i][<NUM_LIT:1>])<EOL>advanceList[i] = True<EOL><DEDENT>else:<EOL><INDENT>advanceList[i] = False<EOL>retGroups.append(None)<EOL><DEDENT><DEDENT>yield tuple(retGroups)<EOL><DEDENT>

Like itertools.groupby, with the following additions:

    - Supports multiple sequences. Instead of returning (k, g), each iteration
      returns (k, g0, g1, ...), with one `g` for each input sequence. The value of
      each `g` is either a non-empty iterator or `None`.
    - It treats the value `None` as an empty sequence. So you can make subsequent
      calls to groupby2 on any `g` value.

    .. note:: Read up on groupby here:
           https://docs.python.org/dev/library/itertools.html#itertools.groupby

    :param args: (list) Parameters alternating between sorted lists and their
                        respective key functions. The lists should be sorted with
                        respect to their key function.

    :returns: (tuple) A n + 1 dimensional tuple, where the first element is the
                    key of the iteration, and the other n entries are groups of
                    objects that share this key. Each group corresponds to the an
                    input sequence. `groupby2` is a generator that returns a tuple
                    for every iteration. If an input sequence has no members with
                    the current key, None is returned in place of a generator.


self.__logItems.append(item)<EOL>return<EOL>

Add an item to the log items list for the currently running session.
        Our self.myAssertXXXXXX wrappers add the current items to the msg that is
        passed to the unittest's assertXXXXX methods.  The extra info will show up
        in test results if the test fails.


print()<EOL>print("<STR_LIT>")<EOL>print(msg % args)<EOL>print("<STR_LIT>" % (datetime.utcnow(), self,), file=sys.stdout)<EOL>print("<STR_LIT>")<EOL>sys.stdout.flush()<EOL>return<EOL>


def assertEqual(self, first, second, msg=None):

unittest.TestCase.assertEqual(self, first, second, self.__wrapMsg(msg))<EOL>return<EOL>

unittest.TestCase.assertEqual override; adds extra log items to msg


Given an instance of a python spatial_pooler return an instance of the CPP
spatial_pooler with identical parameters.


if (imp == "<STR_LIT>"):<EOL><INDENT>spClass = PySpatialPooler<EOL><DEDENT>elif (imp == "<STR_LIT>"):<EOL><INDENT>spClass = CPPSpatialPooler<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>print(params)<EOL>sp = spClass(**params)<EOL>return sp<EOL>

Helper class for creating an instance of the appropriate spatial pooler using
given parameters.

Parameters:
----------------------------
imp:       Either 'py' or 'cpp' for creating the appropriate instance.
params:    A dict for overriding constructor parameters. The keys must
           correspond to contructor parameter names.

Returns the SP object.


seed = int((time.time()%<NUM_LIT>)*<NUM_LIT:10>)<EOL>print("<STR_LIT>", seed, "<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>callStack = traceback.extract_stack(limit=<NUM_LIT:3>)<EOL>print(callStack[<NUM_LIT:0>][<NUM_LIT:2>], "<STR_LIT>", callStack[<NUM_LIT:0>][<NUM_LIT:1>], "<STR_LIT>", callStack[<NUM_LIT:1>][<NUM_LIT:2>])<EOL>return seed<EOL>

Generate and log a 32-bit compatible seed value.


numColumns = sourceSP.getNumColumns()<EOL>numInputs = sourceSP.getNumInputs()<EOL>for i in range(numColumns):<EOL><INDENT>potential = numpy.zeros(numInputs).astype(uintType)<EOL>sourceSP.getPotential(i, potential)<EOL>destSP.setPotential(i, potential)<EOL>perm = numpy.zeros(numInputs).astype(realType)<EOL>sourceSP.getPermanence(i, perm)<EOL>destSP.setPermanence(i, perm)<EOL><DEDENT>

Transfer the permanences from source to dest SP's. This is used in test
routines to counteract some drift between implementations.
We assume the two SP's have identical configurations/parameters.


Implement this method to provide the pattern machine.


params = self._computeTMParams(overrides)<EOL>class MonitoredTemporalMemory(TemporalMemoryMonitorMixin,<EOL>self.getTMClass()): pass<EOL>self.tm = MonitoredTemporalMemory(**params)<EOL>

Initialize Temporal Memory, and other member variables.

:param overrides: overrides for default Temporal Memory parameters


Implement this method to specify the Temporal Memory class.


def title(s=None, additional='<STR_LIT>', stream=sys.stdout):

if s is None:<EOL><INDENT>callable_name, file_name, class_name = getCallerInfo(<NUM_LIT:2>)<EOL>s = callable_name<EOL>if class_name is not None:<EOL><INDENT>s = class_name + '<STR_LIT:.>' + callable_name<EOL><DEDENT><DEDENT>lines = (s + additional).split('<STR_LIT:\n>')<EOL>length = max(len(line) for line in lines)<EOL>print >> stream, '<STR_LIT:->' * length<EOL>print >> stream, s + additional<EOL>print >> stream, '<STR_LIT:->' * length<EOL>

Utility function to display nice titles

    It automatically extracts the name of the function/method it is called from
    and you can add additional text. title() will then print the name
    of the function/method and the additional text surrounded by tow lines
    of dashes. If you don't want the name of the function, you can provide
    alternative text (regardless of the additional text)

    :param s: (string) text to display, uses the function name and arguments by
           default
    :param additional: (string) extra text to display (not needed if s is not
           None)
    :param stream: (stream) the stream to print to. Ny default goes to standard
           output

    Examples:

    .. code-block:: python

      def foo():
        title()

    will display:

    .. code-block:: text

      ---
      foo
      ---

    .. code-block:: python

      def foo():
        title(additional='(), this is cool!!!')

    will display:

    .. code-block:: text

      ----------------------
      foo(), this is cool!!!
      ----------------------

    .. code-block:: python

      def foo():
        title('No function name here!')

    will display:

    .. code-block:: text

      ----------------------
      No function name here!
      ----------------------


<EOL>argspec = inspect.getargspec(f)<EOL>docstring = f.__doc__<EOL>descriptions = {}<EOL>if docstring:<EOL><INDENT>lines = docstring.split('<STR_LIT:\n>')<EOL>i = <NUM_LIT:0><EOL>while i < len(lines):<EOL><INDENT>stripped = lines[i].lstrip()<EOL>if not stripped:<EOL><INDENT>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>indentLevel = lines[i].index(stripped[<NUM_LIT:0>])<EOL>firstWord = stripped.split()[<NUM_LIT:0>]<EOL>if firstWord.endswith('<STR_LIT::>'):<EOL><INDENT>firstWord = firstWord[:-<NUM_LIT:1>]<EOL><DEDENT>if firstWord in argspec.args:<EOL><INDENT>argName = firstWord<EOL>restOfLine = stripped[len(firstWord)+<NUM_LIT:1>:].strip()<EOL>argLines = [restOfLine]<EOL>i += <NUM_LIT:1><EOL>while i < len(lines):<EOL><INDENT>stripped = lines[i].lstrip()<EOL>if not stripped:<EOL><INDENT>break<EOL><DEDENT>if lines[i].index(stripped[<NUM_LIT:0>]) <= indentLevel:<EOL><INDENT>break<EOL><DEDENT>argLines.append(lines[i].strip())<EOL>i += <NUM_LIT:1><EOL><DEDENT>descriptions[argName] = '<STR_LIT:U+0020>'.join(argLines)<EOL><DEDENT>else:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>args = []<EOL>if argspec.defaults:<EOL><INDENT>defaultCount = len(argspec.defaults)<EOL><DEDENT>else:<EOL><INDENT>defaultCount = <NUM_LIT:0><EOL><DEDENT>nonDefaultArgCount = len(argspec.args) - defaultCount<EOL>for i, argName in enumerate(argspec.args):<EOL><INDENT>if i >= nonDefaultArgCount:<EOL><INDENT>defaultValue = argspec.defaults[i - nonDefaultArgCount]<EOL>args.append((argName, descriptions.get(argName, "<STR_LIT>"), defaultValue))<EOL><DEDENT>else:<EOL><INDENT>args.append((argName, descriptions.get(argName, "<STR_LIT>")))<EOL><DEDENT><DEDENT>return args<EOL>

Get the arguments, default values, and argument descriptions for a function.

Parses the argument descriptions out of the function docstring, using a
format something lke this:

::

  [junk]
  argument_name:     description...
    description...
    description...
  [junk]
  [more arguments]

It will find an argument as long as the exact argument name starts the line.
It will then strip a trailing colon, if present, then strip the rest of the
line and use it to start the description. It will then strip and append any
subsequent lines with a greater indent level than the original argument name.

:param f: (function) to inspect
:returns: (list of tuples) (``argName``, ``argDescription``, ``defaultValue``)
  If an argument has no default value, the tuple is only two elements long (as
  ``None`` cannot be used, since it could be a default value itself).


<EOL>dividendMonthSec = aggregationToMonthsSeconds(dividend)<EOL>divisorMonthSec = aggregationToMonthsSeconds(divisor)<EOL>if (dividendMonthSec['<STR_LIT>'] != <NUM_LIT:0> and divisorMonthSec['<STR_LIT>'] != <NUM_LIT:0>)or (dividendMonthSec['<STR_LIT>'] != <NUM_LIT:0> and divisorMonthSec['<STR_LIT>'] != <NUM_LIT:0>):<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if dividendMonthSec['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>return float(dividendMonthSec['<STR_LIT>']) / divisor['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>return float(dividendMonthSec['<STR_LIT>']) / divisorMonthSec['<STR_LIT>']<EOL><DEDENT>

Return the result from dividing two dicts that represent date and time.

Both dividend and divisor are dicts that contain one or more of the following
keys: 'years', 'months', 'weeks', 'days', 'hours', 'minutes', seconds',
'milliseconds', 'microseconds'.

For example:

::

  aggregationDivide({'hours': 4}, {'minutes': 15}) == 16

:param dividend: (dict) The numerator, as a dict representing a date and time
:param divisor: (dict) the denominator, as a dict representing a date and time
:returns: (float) number of times divisor goes into dividend


if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>envValue = os.environ.get("<STR_LIT>" % (cls.envPropPrefix,<EOL>prop.replace('<STR_LIT:.>', '<STR_LIT:_>')), None)<EOL>if envValue is not None:<EOL><INDENT>return envValue<EOL><DEDENT>return cls._properties[prop]<EOL>

Retrieve the requested property as a string. If property does not exist,
        then KeyError will be raised.

        :param prop: (string) name of the property
        :raises: KeyError
        :returns: (string) property value


return int(cls.getString(prop))<EOL>

Retrieve the requested property and return it as an int. If property
        does not exist, then KeyError will be raised.

        :param prop: (string) name of the property
        :returns: (int) property value


if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>result = dict(cls._properties)<EOL>keys = os.environ.keys()<EOL>replaceKeys = filter(lambda x: x.startswith(cls.envPropPrefix),<EOL>keys)<EOL>for envKey in replaceKeys:<EOL><INDENT>key = envKey[len(cls.envPropPrefix):]<EOL>key = key.replace('<STR_LIT:_>', '<STR_LIT:.>')<EOL>result[key] = os.environ[envKey]<EOL><DEDENT>return result<EOL>

Return a dict containing all of the configuration properties

        :returns: (dict) containing all configuration properties.


@classmethod<EOL><INDENT>def _readConfigFile(cls, filename, path=None):<DEDENT>

outputProperties = dict()<EOL>if path is None:<EOL><INDENT>filePath = cls.findConfigFile(filename)<EOL><DEDENT>else:<EOL><INDENT>filePath = os.path.join(path, filename)<EOL><DEDENT>try:<EOL><INDENT>if filePath is not None:<EOL><INDENT>try:<EOL><INDENT>_getLogger().debug("<STR_LIT>", filePath)<EOL>with open(filePath, '<STR_LIT:r>') as inp:<EOL><INDENT>contents = inp.read()<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % filePath)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>contents = resource_string("<STR_LIT>", filename)<EOL><DEDENT>except Exception as resourceException:<EOL><INDENT>if filename in [USER_CONFIG, CUSTOM_CONFIG]:<EOL><INDENT>contents = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>raise resourceException<EOL><DEDENT><DEDENT><DEDENT>elements = ElementTree.XML(contents)<EOL>if elements.tag != '<STR_LIT>':<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (elements.tag))<EOL><DEDENT>propertyElements = elements.findall('<STR_LIT>')<EOL>for propertyItem in propertyElements:<EOL><INDENT>propInfo = dict()<EOL>propertyAttributes = list(propertyItem)<EOL>for propertyAttribute in propertyAttributes:<EOL><INDENT>propInfo[propertyAttribute.tag] = propertyAttribute.text<EOL><DEDENT>name = propInfo.get('<STR_LIT:name>', None)<EOL>if '<STR_LIT:value>' in propInfo and propInfo['<STR_LIT:value>'] is None:<EOL><INDENT>value = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>value = propInfo.get('<STR_LIT:value>', None)<EOL>if value is None:<EOL><INDENT>if '<STR_LIT>' in propInfo:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (str(propInfo)))<EOL><DEDENT><DEDENT><DEDENT>restOfValue = value<EOL>value = '<STR_LIT>'<EOL>while True:<EOL><INDENT>pos = restOfValue.find('<STR_LIT>')<EOL>if pos == -<NUM_LIT:1>:<EOL><INDENT>value += restOfValue<EOL>break<EOL><DEDENT>value += restOfValue[<NUM_LIT:0>:pos]<EOL>varTailPos = restOfValue.find('<STR_LIT:}>', pos)<EOL>if varTailPos == -<NUM_LIT:1>:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (restOfValue))<EOL><DEDENT>varname = restOfValue[pos+<NUM_LIT:6>:varTailPos]<EOL>if varname not in os.environ:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (varname))<EOL><DEDENT>envVarValue = os.environ[varname]<EOL>value += envVarValue<EOL>restOfValue = restOfValue[varTailPos+<NUM_LIT:1>:]<EOL><DEDENT>if name is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (str(propInfo)))<EOL><DEDENT>propInfo['<STR_LIT:value>'] = value<EOL>outputProperties[name] = propInfo<EOL><DEDENT>return outputProperties<EOL><DEDENT>except Exception:<EOL><INDENT>_getLogger().exception("<STR_LIT>",<EOL>filePath)<EOL>raise<EOL><DEDENT>

Parse the given XML file and return a dict describing the file.

        :param filename: (string) name of XML file to parse (no path)
        :param path: (string) path of the XML file. If None, then use the standard
               configuration search path.
        :returns: (dict) with each property as a key and a dict of all the
               property's attributes as value


value = cls.getInt(prop)<EOL>if value not in (<NUM_LIT:0>, <NUM_LIT:1>):<EOL><INDENT>raise ValueError("<STR_LIT>" % (<EOL>value, prop))<EOL><DEDENT>return bool(value)<EOL>

Retrieve the requested property and return it as a bool. If property
        does not exist, then KeyError will be raised. If the property value is
        neither 0 nor 1, then ValueError will be raised

        :param prop: (string) name of the property
        :raises: KeyError, ValueError
        :returns: (bool) property value


@classmethod<EOL><INDENT>def readConfigFile(cls, filename, path=None):<DEDENT>

properties = cls._readConfigFile(filename, path)<EOL>if cls._properties is None:<EOL><INDENT>cls._properties = dict()<EOL><DEDENT>for name in properties:<EOL><INDENT>if '<STR_LIT:value>' in properties[name]:<EOL><INDENT>cls._properties[name] = properties[name]['<STR_LIT:value>']<EOL><DEDENT><DEDENT>

Parse the given XML file and store all properties it describes.

        :param filename: (string) name of XML file to parse (no path)
        :param path: (string) path of the XML file. If None, then use the standard
                      configuration search path.


<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=False)<EOL>

Clear all configuration properties from in-memory cache, but do NOT alter
the custom configuration file. Used in unit-testing.


_getLogger().info("<STR_LIT>"<EOL>"<STR_LIT>", traceback.format_stack())<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=True)<EOL>

Clear all custom configuration settings and delete the persistent custom
configuration store.


cls.readConfigFile(_CustomConfigurationFileWrapper.customFileName)<EOL>

Loads custom configuration settings from their persistent storage.

.. warning :: DO NOT CALL THIS: It's typically not necessary to call this
   method directly. This method exists *solely* for the benefit of
   ``prepare_conf.py``, which needs to load configuration files selectively.


cls._path = os.path.join(os.environ['<STR_LIT>'],<EOL>cls.customFileName)<EOL>

Sets the path of the custom configuration file


return _CustomConfigurationFileWrapper.getCustomDict()<EOL>

returns: (dict) containing all custom configuration properties.


@classmethod<EOL><INDENT>def setCustomProperty(cls, propertyName, value):<DEDENT>

cls.setCustomProperties({propertyName : value})<EOL>

Set a single custom setting and persist it to the custom configuration
store.

:param propertyName: (string) containing the name of the property to get
:param value: (object) value to set the property to


if cls._path is None:<EOL><INDENT>cls._setPath()<EOL><DEDENT>return cls._path<EOL>

Get the path of the custom configuration file


def entryExitLoggingDecorator(func):<EOL><INDENT>@functools.wraps(func)<EOL>def entryExitLoggingWrap(*args, **kwargs):<EOL><INDENT>if entryExitLogLevel is None:<EOL><INDENT>enabled = False<EOL><DEDENT>else:<EOL><INDENT>logger = getLoggerCallback()<EOL>enabled = logger.isEnabledFor(entryExitLogLevel)<EOL><DEDENT>if not enabled:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>funcName = str(func)<EOL>if logArgs:<EOL><INDENT>argsRepr = '<STR_LIT:U+002CU+0020>'.join(<EOL>[repr(a) for a in args] +<EOL>['<STR_LIT>' % (k,v,) for k,v in kwargs.items()])<EOL><DEDENT>else:<EOL><INDENT>argsRepr = '<STR_LIT>'<EOL><DEDENT>logger.log(<EOL>entryExitLogLevel, "<STR_LIT>", funcName, argsRepr,<EOL>'<STR_LIT>' if not logTraceback else '<STR_LIT>' + repr(traceback.format_stack()))<EOL>try:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>finally:<EOL><INDENT>logger.log(<EOL>entryExitLogLevel, "<STR_LIT>", funcName, argsRepr,<EOL>'<STR_LIT>' if not logTraceback else '<STR_LIT>' + repr(traceback.format_stack()))<EOL><DEDENT><DEDENT>return entryExitLoggingWrap<EOL><DEDENT>return entryExitLoggingDecorator<EOL>

Returns a closure suitable for use as function/method decorator for
    logging entry/exit of function/method.

    getLoggerCallback:    user-supplied callback function that takes no args and
                            returns the logger instance to use for logging.
    entryExitLogLevel:    Log level for logging entry/exit of decorated function;
                            e.g., logging.DEBUG; pass None to disable entry/exit
                            logging.
    logArgs:              If True, also log args
    logTraceback:         If True, also log Traceback information

    Usage Examples:
      NOTE: logging must be initialized *before* any loggers are created, else
        there will be no output; see nupic.support.initLogging()

      @logEntryExit()
      def myFunctionBar():
          ...

      @logEntryExit(logTraceback=True)
      @logExceptions()
      def myFunctionGamma():
          ...
          raise RuntimeError("something bad happened")
          ...


logger = (logger if logger is not None else logging.getLogger(__name__))<EOL>def exceptionLoggingDecorator(func):<EOL><INDENT>@functools.wraps(func)<EOL>def exceptionLoggingWrap(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>except:<EOL><INDENT>logger.exception(<EOL>"<STR_LIT>",<EOL>sys.exc_info()[<NUM_LIT:1>], func, '<STR_LIT>'.join(traceback.format_stack()), )<EOL>raise<EOL><DEDENT><DEDENT>return exceptionLoggingWrap<EOL><DEDENT>return exceptionLoggingDecorator<EOL>

Returns a closure suitable for use as function/method decorator for
    logging exceptions that leave the scope of the decorated function. Exceptions
    are logged at ERROR level.

    logger:    user-supplied logger instance. Defaults to logging.getLogger.

    Usage Example:
      NOTE: logging must be initialized *before* any loggers are created, else
        there will be no output; see nupic.support.initLogging()

      @logExceptions()
      def myFunctionFoo():
          ...
          raise RuntimeError("something bad happened")
          ...


return self._recordStore.isCompleted()<EOL>

:returns: True if all records have been read.


return [f.name for f in self._streamFields]<EOL>

Returns all fields in all inputs (list of plain names).

.. note:: currently, only one input is supported


return self._aggMonthsAndSeconds<EOL>

Returns the aggregation period of the record stream as a dict
        containing 'months' and 'seconds'. The months is always an integer and
        seconds is a floating point. Only one is allowed to be non-zero at a
        time.

        Will return the aggregation period from this call. This call is
        used by the :meth:`nupic.data.record_stream.RecordStream.getNextRecordDict`
        method to assign a record number to a record given its timestamp and the
        aggregation interval.

        :returns: aggregationPeriod (as a dict) where:

                  - ``months``: number of months in aggregation period
                  - ``seconds``: number of seconds in aggregation period
                    (as a float)


<EOL>self._recordStore.setCompleted(completed)<EOL>

Marks the stream completed (True or False)

:param completed: (bool) is completed or not


return self._recordCount<EOL>

:returns: the index of the record that will be read next from
          :meth:`getNextRecord`.


return self._recordStore.recordsExistAfter(bookmark)<EOL>

:returns: True if there are records left after the  bookmark.


<EOL>while True:<EOL><INDENT>if self._sourceLastRecordIdx is not None  andself._recordStore.getNextRecordIdx() >= self._sourceLastRecordIdx:<EOL><INDENT>preAggValues = None                             <EOL>bookmark = self._recordStore.getBookmark()<EOL><DEDENT>else:<EOL><INDENT>preAggValues = self._recordStore.getNextRecord()<EOL>bookmark = self._recordStore.getBookmark()<EOL><DEDENT>if preAggValues == ():  <EOL><INDENT>if self._eofOnTimeout:<EOL><INDENT>preAggValues = None  <EOL><DEDENT>else:<EOL><INDENT>return preAggValues  <EOL><DEDENT><DEDENT>self._logger.debug('<STR_LIT>',<EOL>self._recordStore.getNextRecordIdx()-<NUM_LIT:1>, preAggValues)<EOL>(fieldValues, aggBookmark) = self._aggregator.next(preAggValues, bookmark)<EOL>if fieldValues is not None:<EOL><INDENT>self._aggBookmark = aggBookmark<EOL><DEDENT>if preAggValues is None and fieldValues is None:<EOL><INDENT>return None<EOL><DEDENT>if fieldValues is not None:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if self._needFieldsFiltering:<EOL><INDENT>values = []<EOL>srcDict = dict(zip(self._recordStoreFieldNames, fieldValues))<EOL>for name in self._streamFieldNames:<EOL><INDENT>values.append(srcDict[name])<EOL><DEDENT>fieldValues = values<EOL><DEDENT>if self._writer is not None:<EOL><INDENT>self._writer.appendRecord(fieldValues)<EOL><DEDENT>self._recordCount += <NUM_LIT:1><EOL>self._logger.debug('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self._recordCount-<NUM_LIT:1>, fieldValues, self._aggBookmark)<EOL>return fieldValues<EOL>

Returns combined data from all sources (values only).

        :returns: None on EOF; empty sequence on timeout.


different = False<EOL>resultDict = dict()<EOL>resultDict['<STR_LIT>'] = set(da) - set(db)<EOL>if resultDict['<STR_LIT>']:<EOL><INDENT>different = True<EOL><DEDENT>resultDict['<STR_LIT>'] = set(db) - set(da)<EOL>if resultDict['<STR_LIT>']:<EOL><INDENT>different = True<EOL><DEDENT>resultDict['<STR_LIT>'] = []<EOL>for key in (set(da) - resultDict['<STR_LIT>']):<EOL><INDENT>comparisonResult = da[key] == db[key]<EOL>if isinstance(comparisonResult, bool):<EOL><INDENT>isEqual = comparisonResult<EOL><DEDENT>else:<EOL><INDENT>isEqual = comparisonResult.all()<EOL><DEDENT>if not isEqual:<EOL><INDENT>resultDict['<STR_LIT>'].append(key)<EOL>different = True<EOL><DEDENT><DEDENT>assert (((resultDict['<STR_LIT>'] or resultDict['<STR_LIT>'] or<EOL>resultDict['<STR_LIT>']) and different) or not different)<EOL>return resultDict if different else None<EOL>

Compares two python dictionaries at the top level and return differences

    da:             first dictionary
    db:             second dictionary

    Returns:        None if dictionaries test equal; otherwise returns a
                    dictionary as follows:
                    {
                      'inAButNotInB':
                          <sequence of keys that are in da but not in db>
                      'inBButNotInA':
                          <sequence of keys that are in db but not in da>
                      'differentValues':
                          <sequence of keys whose corresponding values differ
                           between da and db>
                    }


remainingDicts = [(d, ())]<EOL>while len(remainingDicts) > <NUM_LIT:0>:<EOL><INDENT>current, prevKeys = remainingDicts.pop()<EOL>for k, v in current.items():<EOL><INDENT>keys = prevKeys + (k,)<EOL>if isinstance(v, dict):<EOL><INDENT>remainingDicts.insert(<NUM_LIT:0>, (v, keys))<EOL><DEDENT>else:<EOL><INDENT>f(v, keys)<EOL><DEDENT><DEDENT><DEDENT>

Recursively applies f to the values in dict d.

    Args:
      d: The dict to recurse over.
      f: A function to apply to values in d that takes the value and a list of
          keys from the root of the dict to the value.


def sort(filename, key, outputFile, fields=None, watermark=<NUM_LIT> * <NUM_LIT> * <NUM_LIT:100>):

if fields is not None:<EOL><INDENT>assert set(key).issubset(set([f[<NUM_LIT:0>] for f in fields]))<EOL><DEDENT>with FileRecordStream(filename) as f:<EOL><INDENT>if fields:<EOL><INDENT>fieldNames = [ff[<NUM_LIT:0>] for ff in fields]<EOL>indices = [f.getFieldNames().index(name) for name in fieldNames]<EOL>assert len(indices) == len(fields)<EOL><DEDENT>else:<EOL><INDENT>fileds = f.getFields()<EOL>fieldNames = f.getFieldNames()<EOL>indices = None<EOL><DEDENT>key = [fieldNames.index(name) for name in key]<EOL>chunk = <NUM_LIT:0><EOL>records = []<EOL>for i, r in enumerate(f):<EOL><INDENT>if indices:<EOL><INDENT>temp = []<EOL>for i in indices:<EOL><INDENT>temp.append(r[i])<EOL><DEDENT>r = temp<EOL><DEDENT>records.append(r)<EOL>available_memory = psutil.avail_phymem()<EOL>if available_memory < watermark:<EOL><INDENT>_sortChunk(records, key, chunk, fields)<EOL>records = []<EOL>chunk += <NUM_LIT:1><EOL><DEDENT><DEDENT>if len(records) > <NUM_LIT:0>:<EOL><INDENT>_sortChunk(records, key, chunk, fields)<EOL>chunk += <NUM_LIT:1><EOL><DEDENT>_mergeFiles(key, chunk, outputFile, fields)<EOL><DEDENT>

Sort a potentially big file

    filename - the input file (standard File format)
    key - a list of field names to sort by
    outputFile - the name of the output file
    fields - a list of fields that should be included (all fields if None)
    watermark - when available memory goes bellow the watermark create a new chunk

    sort() works by reading as records from the file into memory
    and calling _sortChunk() on each chunk. In the process it gets
    rid of unneeded fields if any. Once all the chunks have been sorted and
    written to chunk files it calls _merge() to merge all the chunks into a
    single sorted file.

    Note, that sort() gets a key that contains field names, which it converts
    into field indices for _sortChunk() becuase _sortChunk() doesn't need to know
    the field name.

    sort() figures out by itself how many chunk files to use by reading records
    from the file until the low watermark value of availabel memory is hit and
    then it sorts the current records, generates a chunk file, clears the sorted
    records and starts on a new chunk.

    The key field names are turned into indices


def _mergeFiles(key, chunkCount, outputFile, fields):

title()<EOL>files = [FileRecordStream('<STR_LIT>' % i) for i in range(chunkCount)]<EOL>with FileRecordStream(outputFile, write=True, fields=fields) as o:<EOL><INDENT>files = [FileRecordStream('<STR_LIT>' % i) for i in range(chunkCount)]<EOL>records = [f.getNextRecord() for f in files]<EOL>while not all(r is None for r in records):<EOL><INDENT>indices = [i for i,r in enumerate(records) if r is not None]<EOL>records = [records[i] for i in indices]<EOL>files = [files[i] for i in indices]<EOL>r = min(records, key=itemgetter(*key))<EOL>o.appendRecord(r)<EOL>index = records.index(r)<EOL>records[index] = files[index].getNextRecord()<EOL><DEDENT><DEDENT>for i, f in enumerate(files):<EOL><INDENT>f.close()<EOL>os.remove('<STR_LIT>' % i)<EOL><DEDENT>

Merge sorted chunk files into a sorted output file

    chunkCount - the number of available chunk files
    outputFile the name of the sorted output file

    _mergeFiles()


self._stats = None<EOL>


<EOL>super(FileRecordStream, self).rewind()<EOL>self.close()<EOL>self._file = open(self._filename, self._mode)<EOL>self._reader = csv.reader(self._file, dialect="<STR_LIT>")<EOL>self._reader.next()<EOL>self._reader.next()<EOL>self._reader.next()<EOL>self._recordCount = <NUM_LIT:0><EOL>

Put us back at the beginning of the file again.


<EOL>newSequence = False<EOL>sequenceId = (r[self._sequenceIdIdx]<EOL>if self._sequenceIdIdx is not None else None)<EOL>if sequenceId != self._currSequence:<EOL><INDENT>if sequenceId in self._sequences:<EOL><INDENT>raise Exception('<STR_LIT>' %(sequenceId, r))<EOL><DEDENT>self._sequences.add(self._currSequence)<EOL>self._currSequence = sequenceId<EOL>if self._resetIdx:<EOL><INDENT>assert r[self._resetIdx] == <NUM_LIT:1><EOL><DEDENT>newSequence = True<EOL><DEDENT>else:<EOL><INDENT>reset = False<EOL>if self._resetIdx:<EOL><INDENT>reset = r[self._resetIdx]<EOL>if reset == <NUM_LIT:1>:<EOL><INDENT>newSequence = True<EOL><DEDENT><DEDENT><DEDENT>if not newSequence:<EOL><INDENT>if self._timeStampIdx and self._currTime is not None:<EOL><INDENT>t = r[self._timeStampIdx]<EOL>if t < self._currTime:<EOL><INDENT>raise Exception('<STR_LIT>' % r)<EOL><DEDENT><DEDENT><DEDENT>if self._timeStampIdx:<EOL><INDENT>self._currTime = r[self._timeStampIdx]<EOL><DEDENT>

Keep track of sequence and make sure time goes forward

        Check if the current record is the beginning of a new sequence
        A new sequence starts in 2 cases:

        1. The sequence id changed (if there is a sequence id field)
        2. The reset field is 1 (if there is a reset field)

        Note that if there is no sequenceId field or resetId field then the entire
        dataset is technically one big sequence. The function will not return True
        for the first record in this case. This is Ok because it is important to
        detect new sequences only when there are multiple sequences in the file.


<EOL>if self._mode == self._FILE_WRITE_MODE:<EOL><INDENT>self._file.flush()<EOL><DEDENT>return sum(<NUM_LIT:1> for line in open(self._filename, self._FILE_READ_MODE))<EOL>

Returns:  count of ALL lines in dataset, including header lines


return True<EOL>

Not implemented. CSV file is always considered completed.


def __exit__(self, yupe, value, traceback):

self.close()<EOL>

Context guard - exit

        Ensures that the file is always closed at the end of the 'with' block.
        Lets exceptions propagate.


record = self.getNextRecord()<EOL>if record is None:<EOL><INDENT>raise StopIteration<EOL><DEDENT>return record<EOL>


bookMarkDict = json.loads(bookmark)<EOL>realpath = os.path.realpath(self._filename)<EOL>bookMarkFile = bookMarkDict.get('<STR_LIT>', None)<EOL>if bookMarkFile != realpath:<EOL><INDENT>print ("<STR_LIT>"<EOL>"<STR_LIT>") % (<EOL>realpath, bookMarkDict)<EOL>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>return bookMarkDict['<STR_LIT>']<EOL><DEDENT>

Extracts start row from the bookmark information


return [f.name for f in self._fields]<EOL>

:returns: (list) field names associated with the data.


assert self._file is not None<EOL>assert self._mode == self._FILE_READ_MODE<EOL>try:<EOL><INDENT>line = self._reader.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>if self.rewindAtEOF:<EOL><INDENT>if self._recordCount == <NUM_LIT:0>:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>" % self._filename)<EOL><DEDENT>self.rewind()<EOL>line = self._reader.next()<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>self._recordCount += <NUM_LIT:1><EOL>record = []<EOL>for i, f in enumerate(line):<EOL><INDENT>if f in self._missingValues:<EOL><INDENT>record.append(SENTINEL_VALUE_FOR_MISSING_DATA)<EOL><DEDENT>else:<EOL><INDENT>record.append(self._adapters[i](f))<EOL><DEDENT><DEDENT>return record<EOL>

Returns next available data record from the file.

        :returns: a data row (a list or tuple) if available; None, if no more
                  records in the table (End of Stream - EOS); empty sequence (list
                  or tuple) when timing out while waiting for the next record.


if self._write and self._recordCount==<NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>rowDict = dict(filepath=os.path.realpath(self._filename),<EOL>currentRow=self._recordCount)<EOL>return json.dumps(rowDict)<EOL>

Gets a bookmark or anchor to the current position.

:returns: an anchor to the current position in the data. Passing this
          anchor to a constructor makes the current position to be the first
          returned record.


numLines = self._getTotalLineCount()<EOL>if numLines == <NUM_LIT:0>:<EOL><INDENT>assert self._mode == self._FILE_WRITE_MODE and self._recordCount == <NUM_LIT:0><EOL>numDataRows = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>numDataRows = numLines - self._NUM_HEADER_ROWS<EOL><DEDENT>assert numDataRows >= <NUM_LIT:0><EOL>return numDataRows<EOL>

:returns: (int) count of data rows in dataset (excluding header lines)


return None<EOL>

Not implemented. CSV file version does not provide storage for the error
information


return self._recordCount<EOL>

:returns: (int) the index of the record that will be read next from
          :meth:`~.FileRecordStream.getNextRecord`.


return self<EOL>

Support for the iterator protocol. Return itself


params = None<EOL>if isinstance(funcName, str):<EOL><INDENT>if funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_sum<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_first<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_last<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_mean<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = max<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = min<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_mode<EOL><DEDENT>elif funcName.startswith('<STR_LIT>'):<EOL><INDENT>fp = _aggr_weighted_mean<EOL>paramsName = funcName[<NUM_LIT:6>:]<EOL>params = [f[<NUM_LIT:0>] for f in self._inputFields].index(paramsName)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fp = funcName<EOL><DEDENT>return (fp, params)<EOL>

Given the name of an aggregation function, returns the function pointer
        and param.

        Parameters:
        ------------------------------------------------------------------------
        funcName:  a string (name of function) or funcPtr
        retval:   (funcPtr, param)


assert isinstance(t, datetime.datetime)<EOL>if self._aggTimeDelta:<EOL><INDENT>return t + self._aggTimeDelta<EOL><DEDENT>else:<EOL><INDENT>year = t.year + self._aggYears + (t.month - <NUM_LIT:1> + self._aggMonths) / <NUM_LIT:12><EOL>month = (t.month - <NUM_LIT:1> + self._aggMonths) % <NUM_LIT:12> + <NUM_LIT:1><EOL>return t.replace(year=year, month=month)<EOL><DEDENT>

Add the aggregation period to the input time t and return a datetime object

        Years and months are handled as aspecial case due to leap years
        and months with different number of dates. They can't be converted
        to a strict timedelta because a period of 3 months will have different
        durations actually. The solution is to just add the years and months
        fields directly to the current time.

        Other periods are converted to timedelta and just added to current time.


def generateDataset(aggregationInfo, inputFilename, outputFilename=None):

<EOL>inputFullPath = resource_filename("<STR_LIT>", inputFilename)<EOL>inputObj = FileRecordStream(inputFullPath)<EOL>aggregator = Aggregator(aggregationInfo=aggregationInfo,<EOL>inputFields=inputObj.getFields())<EOL>if aggregator.isNullAggregation():<EOL><INDENT>return inputFullPath<EOL><DEDENT>if outputFilename is None:<EOL><INDENT>outputFilename = '<STR_LIT>' %os.path.splitext(os.path.basename(inputFullPath))[<NUM_LIT:0>]<EOL>timePeriods = '<STR_LIT>''<STR_LIT>'<EOL>for k in timePeriods.split():<EOL><INDENT>if aggregationInfo.get(k, <NUM_LIT:0>) > <NUM_LIT:0>:<EOL><INDENT>outputFilename += '<STR_LIT>' % (k, aggregationInfo[k])<EOL><DEDENT><DEDENT>outputFilename += '<STR_LIT>'<EOL>outputFilename = os.path.join(os.path.dirname(inputFullPath), outputFilename)<EOL><DEDENT>lockFilePath = outputFilename + '<STR_LIT>'<EOL>if os.path.isfile(outputFilename) oros.path.isfile(lockFilePath):<EOL><INDENT>while os.path.isfile(lockFilePath):<EOL><INDENT>print('<STR_LIT>' %lockFilePath)<EOL>time.sleep(<NUM_LIT:1>)<EOL><DEDENT>return outputFilename<EOL><DEDENT>lockFD = open(lockFilePath, '<STR_LIT:w>')<EOL>outputObj = FileRecordStream(streamID=outputFilename, write=True,<EOL>fields=inputObj.getFields())<EOL>while True:<EOL><INDENT>inRecord = inputObj.getNextRecord()<EOL>(aggRecord, aggBookmark) = aggregator.next(inRecord, None)<EOL>if aggRecord is None and inRecord is None:<EOL><INDENT>break<EOL><DEDENT>if aggRecord is not None:<EOL><INDENT>outputObj.appendRecord(aggRecord)<EOL><DEDENT><DEDENT>return outputFilename<EOL>

Generate a dataset of aggregated values

    Parameters:
    ----------------------------------------------------------------------------
    aggregationInfo: a dictionary that contains the following entries
      - fields: a list of pairs. Each pair is a field name and an
        aggregation function (e.g. sum). The function will be used to aggregate
        multiple values during the aggregation period.

    aggregation period: 0 or more of unit=value fields; allowed units are:
          [years months] |
          [weeks days hours minutes seconds milliseconds microseconds]
          NOTE: years and months are mutually-exclusive with the other units.
                See getEndTime() and _aggregate() for more details.
          Example1: years=1, months=6,
          Example2: hours=1, minutes=30,
          If none of the period fields are specified or if all that are specified
          have values of 0, then aggregation will be suppressed, and the given
          inputFile parameter value will be returned.

    inputFilename: filename of the input dataset within examples/prediction/data

    outputFilename: name for the output file. If not given, a name will be
          generated based on the input filename and the aggregation params

    retval: Name of the generated output file. This will be the same as the input
        file name if no aggregation needed to be performed

    If the input file contained a time field, sequence id field or reset field
    that were not specified in aggregationInfo fields, those fields will be
    added automatically with the following rules:

    1. The order will be R, S, T, rest of the fields
    2. The aggregation function for all will be to pick the first: lambda x: x[0]

      Returns: the path of the aggregated data file if aggregation was performed
        (in the same directory as the given input file); if aggregation did not
        need to be performed, then the given inputFile argument value is returned.


record = []<EOL>for i, (fieldIdx, aggFP, paramIdx) in enumerate(self._fields):<EOL><INDENT>if aggFP is None: <EOL><INDENT>continue<EOL><DEDENT>values = self._slice[i]<EOL>refIndex = None<EOL>if paramIdx is not None:<EOL><INDENT>record.append(aggFP(values, self._slice[paramIdx]))<EOL><DEDENT>else:<EOL><INDENT>record.append(aggFP(values))<EOL><DEDENT><DEDENT>return record<EOL>


<EOL>inputFile = resource_filename("<STR_LIT>", inputFile)<EOL>a = defaultdict(lambda: <NUM_LIT:0>, aggregationInfo)<EOL>outputDir = os.path.dirname(inputFile)<EOL>outputFile = '<STR_LIT>' % os.path.splitext(os.path.basename(inputFile))[<NUM_LIT:0>]<EOL>noAggregation = True<EOL>timePeriods = '<STR_LIT>''<STR_LIT>'<EOL>for k in timePeriods.split():<EOL><INDENT>if a[k] > <NUM_LIT:0>:<EOL><INDENT>noAggregation = False<EOL>outputFile += '<STR_LIT>' % (k, a[k])<EOL><DEDENT><DEDENT>if noAggregation:<EOL><INDENT>return inputFile<EOL><DEDENT>outputFile += '<STR_LIT>'<EOL>outputFile = os.path.join(outputDir, outputFile)<EOL>return outputFile<EOL>

Generate the filename for aggregated dataset

    The filename is based on the input filename and the
    aggregation period.

    Returns the inputFile if no aggregation required (aggregation
    info has all 0's)


:returns: storage stats (like min and max values of the fields).


values = self.getNextRecord()<EOL>if values is None:<EOL><INDENT>return None<EOL><DEDENT>if not values:<EOL><INDENT>return dict()<EOL><DEDENT>if self._modelRecordEncoder is None:<EOL><INDENT>self._modelRecordEncoder = ModelRecordEncoder(<EOL>fields=self.getFields(),<EOL>aggregationPeriod=self.getAggregationMonthsAndSeconds())<EOL><DEDENT>return self._modelRecordEncoder.encode(values)<EOL>

Returns next available data record from the storage as a dict, with the
        keys being the field names. This also adds in some meta fields:

          - ``_category``: The value from the category field (if any)
          - ``_reset``: True if the reset field was True (if any)
          - ``_sequenceId``: the value from the sequenceId field (if any)


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.sequence)<EOL>

:returns: (int) index of the ``sequenceId`` field.


Marks the stream completed.

:param completed: (bool) is completed?


self._sequenceId = -<NUM_LIT:1><EOL>

Put us back at the beginning of the file again


@abstractmethod<EOL><INDENT>def appendRecords(self, records, progressCB=None):<DEDENT>

Saves multiple records in the underlying storage. Should be implemented in
subclasses.

:param records: (list) of objects to store
:param progressCB: (func) called after each appension


for i, field in enumerate(fields):<EOL><INDENT>if field.special == special:<EOL><INDENT>return i<EOL><DEDENT><DEDENT>return None<EOL>

Return index of the field matching the field meta special value.
    :param fields: sequence of nupic.data.fieldmeta.FieldMetaInfo objects
      representing the fields of a stream
    :param special: one of the special field attribute values from
      nupic.data.fieldmeta.FieldMetaSpecial
    :returns: first zero-based index of the field tagged with the target field
      meta special attribute; None if no such field


<EOL>result = dict(zip(self._fieldNames, inputRow))<EOL>if self._categoryFieldIndex is not None:<EOL><INDENT>if isinstance(inputRow[self._categoryFieldIndex], int):<EOL><INDENT>result['<STR_LIT>'] = [inputRow[self._categoryFieldIndex]]<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = (inputRow[self._categoryFieldIndex]<EOL>if inputRow[self._categoryFieldIndex]<EOL>else [None])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = [None]<EOL><DEDENT>if self._resetFieldIndex is not None:<EOL><INDENT>result['<STR_LIT>'] = int(bool(inputRow[self._resetFieldIndex]))<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = <NUM_LIT:0><EOL><DEDENT>if self._learningFieldIndex is not None:<EOL><INDENT>result['<STR_LIT>'] = int(bool(inputRow[self._learningFieldIndex]))<EOL><DEDENT>result['<STR_LIT>'] = None<EOL>if self._timestampFieldIndex is not None:<EOL><INDENT>result['<STR_LIT>'] = inputRow[self._timestampFieldIndex]<EOL>result['<STR_LIT>'] = self._computeTimestampRecordIdx(<EOL>inputRow[self._timestampFieldIndex])<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = None<EOL><DEDENT>hasReset = self._resetFieldIndex is not None<EOL>hasSequenceId = self._sequenceFieldIndex is not None<EOL>if hasReset and not hasSequenceId:<EOL><INDENT>if result['<STR_LIT>']:<EOL><INDENT>self._sequenceId += <NUM_LIT:1><EOL><DEDENT>sequenceId = self._sequenceId<EOL><DEDENT>elif not hasReset and hasSequenceId:<EOL><INDENT>sequenceId = inputRow[self._sequenceFieldIndex]<EOL>result['<STR_LIT>'] = int(sequenceId != self._sequenceId)<EOL>self._sequenceId = sequenceId<EOL><DEDENT>elif hasReset and hasSequenceId:<EOL><INDENT>sequenceId = inputRow[self._sequenceFieldIndex]<EOL><DEDENT>else:<EOL><INDENT>sequenceId = <NUM_LIT:0><EOL><DEDENT>if sequenceId is not None:<EOL><INDENT>result['<STR_LIT>'] = hash(sequenceId)<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = None<EOL><DEDENT>return result<EOL>

Encodes the given input row as a dict, with the
        keys being the field names. This also adds in some meta fields:
          '_category': The value from the category field (if any)
          '_reset': True if the reset field was True (if any)
          '_sequenceId': the value from the sequenceId field (if any)

        :param inputRow: sequence of values corresponding to a single input metric
          data row
        :rtype: dict


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.reset)<EOL>

:returns: (int) index of the ``reset`` field; ``None`` if no such field.


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.timestamp)<EOL>

:returns: (int) index of the ``timestamp`` field.


if self._aggregationPeriod is None:<EOL><INDENT>return None<EOL><DEDENT>if self._aggregationPeriod['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>assert self._aggregationPeriod['<STR_LIT>'] == <NUM_LIT:0><EOL>result = int(<EOL>(recordTS.year * <NUM_LIT:12> + (recordTS.month-<NUM_LIT:1>)) /<EOL>self._aggregationPeriod['<STR_LIT>'])<EOL><DEDENT>elif self._aggregationPeriod['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>delta = recordTS - datetime.datetime(year=<NUM_LIT:1>, month=<NUM_LIT:1>, day=<NUM_LIT:1>)<EOL>deltaSecs = delta.days * <NUM_LIT> * <NUM_LIT> * <NUM_LIT>+ delta.seconds+ delta.microseconds / <NUM_LIT><EOL>result = int(deltaSecs / self._aggregationPeriod['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>result = None<EOL><DEDENT>return result<EOL>

Give the timestamp of a record (a datetime object), compute the record's
        timestamp index - this is the timestamp divided by the aggregation period.

        Parameters:
        ------------------------------------------------------------------------
        recordTS:  datetime instance
        retval:    record timestamp index, or None if no aggregation period


def __init__(self, fields, aggregationPeriod=None):

if not fields:<EOL><INDENT>raise ValueError('<STR_LIT>' % (fields,))<EOL><DEDENT>self._fields = fields<EOL>self._aggregationPeriod = aggregationPeriod<EOL>self._sequenceId = -<NUM_LIT:1><EOL>self._fieldNames = tuple(f.name for f in fields)<EOL>self._categoryFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.category)<EOL>self._resetFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.reset)<EOL>self._sequenceFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.sequence)<EOL>self._timestampFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.timestamp)<EOL>self._learningFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.learning)<EOL>

:param fields: non-empty sequence of nupic.data.fieldmeta.FieldMetaInfo
  objects corresponding to fields in input rows.
:param aggregationPeriod: (dict) aggregation period of the record stream
  containing 'months' and 'seconds'. The months is always an integer
  and seconds is a floating point. Only one is allowed to be non-zero at a
  time. If there is no aggregation associated with the stream, pass None.
  Typically, a raw file or hbase stream will NOT have any aggregation info,
  but subclasses of RecordStreamIface, like StreamReader, will and will
  provide the aggregation period. This is used by the encode method to
  assign a record number to a record given its timestamp and the aggregation
  interval.


stats = self.getStats()<EOL>if stats == None:<EOL><INDENT>return None<EOL><DEDENT>minValues = stats.get('<STR_LIT>', None)<EOL>if minValues == None:<EOL><INDENT>return None<EOL><DEDENT>index = self.getFieldNames().index(fieldName)<EOL>return minValues[index]<EOL>

If underlying implementation does not support min/max stats collection,
or if a field type does not support min/max (non scalars), the return
value will be None.

:param fieldName: (string) name of field to get min
:returns: current minimum value for the field ``fieldName``.


:returns: (list) of field names associated with the data.


stats = self.getStats()<EOL>if stats == None:<EOL><INDENT>return None<EOL><DEDENT>maxValues = stats.get('<STR_LIT>', None)<EOL>if maxValues == None:<EOL><INDENT>return None<EOL><DEDENT>index = self.getFieldNames().index(fieldName)<EOL>return maxValues[index]<EOL>

If underlying implementation does not support min/max stats collection,
or if a field type does not support min/max (non scalars), the return
value will be None.

:param fieldName: (string) name of field to get max
:returns: current maximum value for the field ``fieldName``.


:returns: (list) of :class:`nupic.data.fieldmeta.FieldMetaInfo` objects for
    each field in the stream. Might be None, if that information is provided
    externally (through the `Stream Definition <stream-def.html>`_,
    for example).


:returns: (int) index of the record that will be read next from
          :meth:`getNextRecord`


Set the read timeout in seconds

:param timeout: (int or floating point)


def generateStats(filename, maxSamples = None,):

<EOL>statsCollectorMapping = {'<STR_LIT:float>':    FloatStatsCollector,<EOL>'<STR_LIT:int>':      IntStatsCollector,<EOL>'<STR_LIT:string>':   StringStatsCollector,<EOL>'<STR_LIT>': DateTimeStatsCollector,<EOL>'<STR_LIT:bool>':     BoolStatsCollector,<EOL>}<EOL>filename = resource_filename("<STR_LIT>", filename)<EOL>print("<STR_LIT:*>"*<NUM_LIT>)<EOL>print("<STR_LIT>" % (filename,))<EOL>dataFile = FileRecordStream(filename)<EOL>statsCollectors = []<EOL>for fieldName, fieldType, fieldSpecial in dataFile.getFields():<EOL><INDENT>statsCollector =statsCollectorMapping[fieldType](fieldName, fieldType, fieldSpecial)<EOL>statsCollectors.append(statsCollector)<EOL><DEDENT>if maxSamples is None:<EOL><INDENT>maxSamples = <NUM_LIT><EOL><DEDENT>for i in range(maxSamples):<EOL><INDENT>record = dataFile.getNextRecord()<EOL>if record is None:<EOL><INDENT>break<EOL><DEDENT>for i, value in enumerate(record):<EOL><INDENT>statsCollectors[i].addValue(value)<EOL><DEDENT><DEDENT>stats = {}<EOL>for statsCollector in statsCollectors:<EOL><INDENT>statsCollector.getStats(stats)<EOL><DEDENT>if dataFile.getResetFieldIdx() is not None:<EOL><INDENT>resetFieldName,_,_ = dataFile.getFields()[dataFile.reset]<EOL>stats.pop(resetFieldName)<EOL><DEDENT>if VERBOSITY > <NUM_LIT:0>:<EOL><INDENT>pprint.pprint(stats)<EOL><DEDENT>return stats<EOL>

Collect statistics for each of the fields in the user input data file and
return a stats dict object.

Parameters:
------------------------------------------------------------------------------
filename:             The path and name of the data file.
maxSamples:           Upper bound on the number of rows to be processed
retval:               A dictionary of dictionaries. The top level keys are the
                      field names and the corresponding values are the statistics
                      collected for the individual file.
                      Example:
                      {
                        'consumption':{'min':0,'max':90,'mean':50,...},
                        'gym':{'numDistinctCategories':10,...},
                        ...
                       }