altitude = None<EOL>if len(inputData) == <NUM_LIT:4>:<EOL><INDENT>(speed, longitude, latitude, altitude) = inputData<EOL><DEDENT>else:<EOL><INDENT>(speed, longitude, latitude) = inputData<EOL><DEDENT>coordinate = self.coordinateForPosition(longitude, latitude, altitude)<EOL>radius = self.radiusForSpeed(speed)<EOL>super(GeospatialCoordinateEncoder, self).encodeIntoArray(<EOL>(coordinate, radius), output)<EOL>

See `nupic.encoders.base.Encoder` for more information.

:param: inputData (tuple) Contains speed (float), longitude (float),
                         latitude (float), altitude (float)
:param: output (numpy.array) Stores encoded SDR in this numpy array


<EOL>if self._topDownMappingM is None:<EOL><INDENT>if self.periodic:<EOL><INDENT>self._topDownValues = numpy.arange(self.minval + self.resolution / <NUM_LIT>,<EOL>self.maxval,<EOL>self.resolution)<EOL><DEDENT>else:<EOL><INDENT>self._topDownValues = numpy.arange(self.minval,<EOL>self.maxval + self.resolution / <NUM_LIT>,<EOL>self.resolution)<EOL><DEDENT>numCategories = len(self._topDownValues)<EOL>self._topDownMappingM = SM32(numCategories, self.n)<EOL>outputSpace = numpy.zeros(self.n, dtype=GetNTAReal())<EOL>for i in range(numCategories):<EOL><INDENT>value = self._topDownValues[i]<EOL>value = max(value, self.minval)<EOL>value = min(value, self.maxval)<EOL>self.encodeIntoArray(value, outputSpace, learn=False)<EOL>self._topDownMappingM.setRowFromDense(i, outputSpace)<EOL><DEDENT><DEDENT>return self._topDownMappingM<EOL>

Return the interal _topDownMappingM matrix used for handling the
        bucketInfo() and topDownCompute() methods. This is a matrix, one row per
        category (bucket) where each row contains the encoded output for that
        category.


<EOL>topDownMappingM = self._getTopDownMapping()<EOL>category = topDownMappingM.rightVecProd(encoded).argmax()<EOL>return self.getBucketInfo([category])<EOL>


if self.w < <NUM_LIT>:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>" % self.w)<EOL><DEDENT>

(helper function) check if the settings are reasonable for SP to work


<EOL>if self._bucketValues is None:<EOL><INDENT>topDownMappingM = self._getTopDownMapping()<EOL>numBuckets = topDownMappingM.nRows()<EOL>self._bucketValues = []<EOL>for bucketIdx in range(numBuckets):<EOL><INDENT>self._bucketValues.append(self.getBucketInfo([bucketIdx])[<NUM_LIT:0>].value)<EOL><DEDENT><DEDENT>return self._bucketValues<EOL>


<EOL>topDownMappingM = self._getTopDownMapping()<EOL>category = buckets[<NUM_LIT:0>]<EOL>encoding = self._topDownMappingM.getRow(category)<EOL>if self.periodic:<EOL><INDENT>inputVal = (self.minval + (self.resolution / <NUM_LIT>) +<EOL>(category * self.resolution))<EOL><DEDENT>else:<EOL><INDENT>inputVal = self.minval + (category * self.resolution)<EOL><DEDENT>return [EncoderResult(value=inputVal, scalar=inputVal, encoding=encoding)]<EOL>


def closenessScores(self, expValues, actValues, fractional=True):

expValue = expValues[<NUM_LIT:0>]<EOL>actValue = actValues[<NUM_LIT:0>]<EOL>if self.periodic:<EOL><INDENT>expValue = expValue % self.maxval<EOL>actValue = actValue % self.maxval<EOL><DEDENT>err = abs(expValue - actValue)<EOL>if self.periodic:<EOL><INDENT>err = min(err, self.maxval - err)<EOL><DEDENT>if fractional:<EOL><INDENT>pctErr = float(err) / (self.maxval - self.minval)<EOL>pctErr = min(<NUM_LIT:1.0>, pctErr)<EOL>closeness = <NUM_LIT:1.0> - pctErr<EOL><DEDENT>else:<EOL><INDENT>closeness = err<EOL><DEDENT>return numpy.array([closeness])<EOL>


def _initEncoder(self, w, minval, maxval, n, radius, resolution):

if n != <NUM_LIT:0>:<EOL><INDENT>if (radius !=<NUM_LIT:0> or resolution != <NUM_LIT:0>):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>assert n > w<EOL>self.n = n<EOL>if (minval is not None and maxval is not None):<EOL><INDENT>if not self.periodic:<EOL><INDENT>self.resolution = float(self.rangeInternal) / (self.n - self.w)<EOL><DEDENT>else:<EOL><INDENT>self.resolution = float(self.rangeInternal) / (self.n)<EOL><DEDENT>self.radius = self.w * self.resolution<EOL>if self.periodic:<EOL><INDENT>self.range = self.rangeInternal<EOL><DEDENT>else:<EOL><INDENT>self.range = self.rangeInternal + self.resolution<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if radius != <NUM_LIT:0>:<EOL><INDENT>if (resolution != <NUM_LIT:0>):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.radius = radius<EOL>self.resolution = float(self.radius) / w<EOL><DEDENT>elif resolution != <NUM_LIT:0>:<EOL><INDENT>self.resolution = float(resolution)<EOL>self.radius = self.resolution * self.w<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if (minval is not None and maxval is not None):<EOL><INDENT>if self.periodic:<EOL><INDENT>self.range = self.rangeInternal<EOL><DEDENT>else:<EOL><INDENT>self.range = self.rangeInternal + self.resolution<EOL><DEDENT>nfloat = self.w * (self.range / self.radius) + <NUM_LIT:2> * self.padding<EOL>self.n = int(math.ceil(nfloat))<EOL><DEDENT><DEDENT>

(helper function)  There are three different ways of thinking about the representation.
         Handle each case here.


if self.ncategories==<NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>topDownMappingM = self._getTopDownMapping()<EOL>categoryIndex = topDownMappingM.rightVecProd(encoded).argmax()<EOL>category = self.categories[categoryIndex]<EOL>encoding = topDownMappingM.getRow(categoryIndex)<EOL>return EncoderResult(value=category, scalar=categoryIndex, encoding=encoding)<EOL>


expValue = expValues[<NUM_LIT:0>]<EOL>actValue = actValues[<NUM_LIT:0>]<EOL>if expValue == actValue:<EOL><INDENT>closeness = <NUM_LIT:1.0><EOL><DEDENT>else:<EOL><INDENT>closeness = <NUM_LIT:0.0><EOL><DEDENT>if not fractional:<EOL><INDENT>closeness = <NUM_LIT:1.0> - closeness<EOL><DEDENT>return numpy.array([closeness])<EOL>

See the function description in base.py

        kwargs will have the keyword "fractional", which is ignored by this encoder


if input == SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return numpy.array([<NUM_LIT:0>])<EOL><DEDENT>index = self.categoryToIndex.get(input, None)<EOL>if index is None:<EOL><INDENT>if self._learningEnabled:<EOL><INDENT>self._addCategory(input)<EOL>index = self.ncategories - <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>index = <NUM_LIT:0><EOL><DEDENT><DEDENT>return numpy.array([index])<EOL>


def _initializeBucketMap(self, maxBuckets, offset):

<EOL>self._maxBuckets = maxBuckets<EOL>self.minIndex = self._maxBuckets / <NUM_LIT:2><EOL>self.maxIndex = self._maxBuckets / <NUM_LIT:2><EOL>self._offset = offset<EOL>self.bucketMap = {}<EOL>def _permutation(n):<EOL><INDENT>r = numpy.arange(n, dtype=numpy.uint32)<EOL>self.random.shuffle(r)<EOL>return r<EOL><DEDENT>self.bucketMap[self.minIndex] = _permutation(self.n)[<NUM_LIT:0>:self.w]<EOL>self.numTries = <NUM_LIT:0><EOL>

Initialize the bucket map assuming the given number of maxBuckets.


def _newRepresentationOK(self, newRep, newIndex):

if newRep.size != self.w:<EOL><INDENT>return False<EOL><DEDENT>if (newIndex < self.minIndex-<NUM_LIT:1>) or (newIndex > self.maxIndex+<NUM_LIT:1>):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>newRepBinary = numpy.array([False]*self.n)<EOL>newRepBinary[newRep] = True<EOL>midIdx = self._maxBuckets/<NUM_LIT:2><EOL>runningOverlap = self._countOverlap(self.bucketMap[self.minIndex], newRep)<EOL>if not self._overlapOK(self.minIndex, newIndex, overlap=runningOverlap):<EOL><INDENT>return False<EOL><DEDENT>for i in range(self.minIndex+<NUM_LIT:1>, midIdx+<NUM_LIT:1>):<EOL><INDENT>newBit = (i-<NUM_LIT:1>)%self.w<EOL>if newRepBinary[self.bucketMap[i-<NUM_LIT:1>][newBit]]:<EOL><INDENT>runningOverlap -= <NUM_LIT:1><EOL><DEDENT>if newRepBinary[self.bucketMap[i][newBit]]:<EOL><INDENT>runningOverlap += <NUM_LIT:1><EOL><DEDENT>if not self._overlapOK(i, newIndex, overlap=runningOverlap):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>for i in range(midIdx+<NUM_LIT:1>, self.maxIndex+<NUM_LIT:1>):<EOL><INDENT>newBit = i%self.w<EOL>if newRepBinary[self.bucketMap[i-<NUM_LIT:1>][newBit]]:<EOL><INDENT>runningOverlap -= <NUM_LIT:1><EOL><DEDENT>if newRepBinary[self.bucketMap[i][newBit]]:<EOL><INDENT>runningOverlap += <NUM_LIT:1><EOL><DEDENT>if not self._overlapOK(i, newIndex, overlap=runningOverlap):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Return True if this new candidate representation satisfies all our overlap
rules. Since we know that neighboring representations differ by at most
one bit, we compute running overlaps.


if ((isinstance(x, float) and math.isnan(x)) or<EOL>x == SENTINEL_VALUE_FOR_MISSING_DATA):<EOL><INDENT>return [None]<EOL><DEDENT>if self._offset is None:<EOL><INDENT>self._offset = x<EOL><DEDENT>bucketIdx = (<EOL>(self._maxBuckets/<NUM_LIT:2>) + int(round((x - self._offset) / self.resolution))<EOL>)<EOL>if bucketIdx < <NUM_LIT:0>:<EOL><INDENT>bucketIdx = <NUM_LIT:0><EOL><DEDENT>elif bucketIdx >= self._maxBuckets:<EOL><INDENT>bucketIdx = self._maxBuckets-<NUM_LIT:1><EOL><DEDENT>return [bucketIdx]<EOL>


if i in self.bucketMap and j in self.bucketMap:<EOL><INDENT>iRep = self.bucketMap[i]<EOL>jRep = self.bucketMap[j]<EOL>return self._countOverlap(iRep, jRep)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Return the overlap between bucket indices i and j


return self.n<EOL>


if index < <NUM_LIT:0>:<EOL><INDENT>index = <NUM_LIT:0><EOL><DEDENT>if index >= self._maxBuckets:<EOL><INDENT>index = self._maxBuckets-<NUM_LIT:1><EOL><DEDENT>if index not in self.bucketMap:<EOL><INDENT>if self.verbosity >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>", index)<EOL><DEDENT>self._createBucket(index)<EOL><DEDENT>return self.bucketMap[index]<EOL>

Given a bucket index, return the list of non-zero bits. If the bucket
index does not exist, it is created. If the index falls outside our range
we clip it.

:param index The bucket index to get non-zero bits for.
@returns numpy array of indices of non-zero bits for specified index.


if seed != -<NUM_LIT:1>:<EOL><INDENT>self.random = NupicRandom(seed)<EOL><DEDENT>else:<EOL><INDENT>self.random = NupicRandom()<EOL><DEDENT>


overlap = <NUM_LIT:0><EOL>for e in rep1:<EOL><INDENT>if e in rep2:<EOL><INDENT>overlap += <NUM_LIT:1><EOL><DEDENT><DEDENT>return overlap<EOL>

Return the overlap between two representations. rep1 and rep2 are lists of
non-zero indices.


def _newRepresentation(self, index, newIndex):

newRepresentation = self.bucketMap[index].copy()<EOL>ri = newIndex % self.w<EOL>newBit = self.random.getUInt32(self.n)<EOL>newRepresentation[ri] = newBit<EOL>while newBit in self.bucketMap[index] ornot self._newRepresentationOK(newRepresentation, newIndex):<EOL><INDENT>self.numTries += <NUM_LIT:1><EOL>newBit = self.random.getUInt32(self.n)<EOL>newRepresentation[ri] = newBit<EOL><DEDENT>return newRepresentation<EOL>

Return a new representation for newIndex that overlaps with the
representation at index by exactly w-1 bits


if self.minval is None or self.maxval is None:<EOL><INDENT>return [EncoderResult(value=<NUM_LIT:0>, scalar=<NUM_LIT:0>,<EOL>encoding=numpy.zeros(self.n))]<EOL><DEDENT>return super(AdaptiveScalarEncoder, self).getBucketInfo(buckets)<EOL>


if self.minval is None or self.maxval is None:<EOL><INDENT>return [EncoderResult(value=<NUM_LIT:0>, scalar=<NUM_LIT:0>,<EOL>encoding=numpy.zeros(self.n))]<EOL><DEDENT>return super(AdaptiveScalarEncoder, self).topDownCompute(encoded)<EOL>


self.slidingWindow.next(input)<EOL>if self.minval is None and self.maxval is None:<EOL><INDENT>self.minval = input<EOL>self.maxval = input+<NUM_LIT:1>   <EOL>self._setEncoderParams()<EOL><DEDENT>elif learn:<EOL><INDENT>sorted = self.slidingWindow.getSlidingWindow()<EOL>sorted.sort()<EOL>minOverWindow = sorted[<NUM_LIT:0>]<EOL>maxOverWindow = sorted[len(sorted)-<NUM_LIT:1>]<EOL>if minOverWindow < self.minval:<EOL><INDENT>if self.verbosity >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>"% (self.name, input, self.minval, minOverWindow))<EOL><DEDENT>self.minval = minOverWindow       <EOL>self._setEncoderParams()<EOL><DEDENT>if maxOverWindow > self.maxval:<EOL><INDENT>if self.verbosity >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>"% (self.name, input, self.maxval, maxOverWindow))<EOL><DEDENT>self.maxval = maxOverWindow       <EOL>self._setEncoderParams()<EOL><DEDENT><DEDENT>

Potentially change the minval and maxval using input.
**The learn flag is currently not supported by cla regions.**


self.rangeInternal = float(self.maxval - self.minval)<EOL>self.resolution = float(self.rangeInternal) / (self.n - self.w)<EOL>self.radius = self.w * self.resolution<EOL>self.range = self.rangeInternal + self.resolution<EOL>self.nInternal = self.n - <NUM_LIT:2> * self.padding<EOL>self._bucketValues = None<EOL>

Set the radius, resolution and range. These values are updated when minval
and/or maxval change.


global g_max_concurrency, g_max_concurrency_raise_exception<EOL>assert g_max_concurrency is not None<EOL>assert self not in self._clsOutstandingInstances, repr(self)<EOL>self._creationTracebackString = traceback.format_stack()<EOL>if self._clsNumOutstanding >= g_max_concurrency:<EOL><INDENT>errorMsg = ("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % (<EOL>self._clsNumOutstanding, g_max_concurrency, self,<EOL>len(self._clsOutstandingInstances), self._clsOutstandingInstances,)<EOL>self._logger.error(errorMsg)<EOL>if g_max_concurrency_raise_exception:<EOL><INDENT>raise ConcurrencyExceededError(errorMsg)<EOL><DEDENT><DEDENT>self._clsOutstandingInstances.add(self)<EOL>self._addedToInstanceSet = True<EOL>return<EOL>

Check for concurrency violation and add self to
        _clsOutstandingInstances.

        ASSUMPTION: Called from constructor BEFORE _clsNumOutstanding is
        incremented


raise NotImplementedError()<EOL>

Get a Connection instance.

        Parameters:
        ----------------------------------------------------------------
        retval:       A ConnectionWrapper instance.
                        Caller is responsible for calling the  ConnectionWrapper
                        instance's release() method to release resources.


logger = logging.getLogger(<EOL>"<STR_LIT:.>".join(['<STR_LIT>', _MODULE_NAME, cls.__name__]))<EOL>if logLevel is not None:<EOL><INDENT>logger.setLevel(logLevel)<EOL><DEDENT>return logger<EOL>

Gets a logger for the given class in this module


self._logger.debug("<STR_LIT>", self)<EOL>if self._addedToInstanceSet:<EOL><INDENT>try:<EOL><INDENT>self._clsOutstandingInstances.remove(self)<EOL><DEDENT>except:<EOL><INDENT>self._logger.exception(<EOL>"<STR_LIT>", self)<EOL>raise<EOL><DEDENT><DEDENT>self._releaser(dbConn=self.dbConn, cursor=self.cursor)<EOL>self.__class__._clsNumOutstanding -= <NUM_LIT:1><EOL>assert self._clsNumOutstanding >= <NUM_LIT:0>,"<STR_LIT>" % (self._clsNumOutstanding,)<EOL>self._releaser = None<EOL>self.cursor = None<EOL>self.dbConn = None<EOL>self._creationTracebackString = None<EOL>self._addedToInstanceSet = False<EOL>self._logger = None<EOL>return<EOL>

Release the database connection and cursor

        The receiver of the Connection instance MUST call this method in order
        to reclaim resources


def _releaseConnection(self, dbConn, cursor):

self._logger.debug("<STR_LIT>")<EOL>cursor.close()<EOL>return<EOL>

Release database connection and cursor; passed as a callback to
        ConnectionWrapper


self._logger.debug("<STR_LIT>")<EOL>cursor.close()<EOL>dbConn.close()<EOL>return<EOL>


if cls._connectionPolicy is None:<EOL><INDENT>logger = _getLogger(cls)<EOL>logger.info("<STR_LIT>",<EOL>cls._connectionPolicyInstanceProvider)<EOL>cls._connectionPolicy = cls._connectionPolicyInstanceProvider()<EOL>logger.debug("<STR_LIT>", cls._connectionPolicy)<EOL><DEDENT>return cls._connectionPolicy.acquireConnection()<EOL>

Acquire a ConnectionWrapper instance that represents a connection
        to the SQL server per nupic.cluster.database.* configuration settings.

        NOTE: caller is responsible for calling the ConnectionWrapper instance's
        release() method after using the connection in order to release resources.
        Better yet, use the returned ConnectionWrapper instance in a Context Manager
        statement for automatic invocation of release():
        Example:
            # If using Jython 2.5.x, first import with_statement at the very top of
            your script (don't need this import for Jython/Python 2.6.x and later):
            from __future__ import with_statement
            # Then:
            from nupic.database.Connection import ConnectionFactory
            # Then use it like this
            with ConnectionFactory.get() as conn:
              conn.cursor.execute("SELECT ...")
              conn.cursor.fetchall()
              conn.cursor.execute("INSERT ...")

        WARNING: DO NOT close the underlying connection or cursor as it may be
        shared by other modules in your process.  ConnectionWrapper's release()
        method will do the right thing.

        Parameters:
        ----------------------------------------------------------------
        retval:       A ConnectionWrapper instance. NOTE: Caller is responsible
                        for releasing resources as described above.


self._logger = _getLogger(self.__class__)<EOL>self._logger.debug("<STR_LIT>")<EOL>self._pool = PooledDB(**_getCommonSteadyDBArgsDict())<EOL>self._logger.info("<STR_LIT>", self.__class__.__name__)<EOL>return<EOL>

Consruct an instance. The instance's open() method must be
        called to make it ready for acquireConnection() calls.


logger = _getLogger(cls)<EOL>logger.debug(<EOL>"<STR_LIT>",<EOL>platform.system(), pymysql.VERSION)<EOL>if platform.system() == "<STR_LIT>":<EOL><INDENT>policy = SingleSharedConnectionPolicy()<EOL><DEDENT>else:<EOL><INDENT>policy = PooledConnectionPolicy()<EOL><DEDENT>return policy<EOL>

[private] Create the default database connection policy instance

        Parameters:
        ----------------------------------------------------------------
        retval:            The default database connection policy instance


Get a Connection instance.

        Parameters:
        ----------------------------------------------------------------
        retval:       A ConnectionWrapper instance. NOTE: Caller
                        is responsible for calling the  ConnectionWrapper
                        instance's release() method or use it in a context manager
                        expression (with ... as:) to release resources.


self._logger.info("<STR_LIT>")<EOL>if self._pool is not None:<EOL><INDENT>self._pool.close()<EOL>self._pool = None<EOL><DEDENT>else:<EOL><INDENT>self._logger.warning(<EOL>"<STR_LIT>")<EOL><DEDENT>return<EOL>

Close the policy instance and its database connection pool.


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobIncrementIntField(self, jobID, fieldName, increment=<NUM_LIT:1>,<EOL>useConnectionID=False):<DEDENT>

<EOL>dbFieldName = self._jobs.pubToDBNameDict[fieldName]<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, dbFieldName, dbFieldName)<EOL>sqlParams = [increment, jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>result = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if result != <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>""<STR_LIT>" % (<EOL>dbFieldName, jobID, self._connectionID, result, query))<EOL><DEDENT>

Incremet the value of 1 field in a job by increment. The 'fieldName' is
        the public name of the field (camelBack, not the lower_case_only form as
        stored in the DB).

        This method is used for example by HypersearcWorkers to update the
        engWorkerState field periodically. By qualifying on curValue, it insures
        that only 1 worker at a time is elected to perform the next scheduled
        periodic sweep of the models.

        Parameters:
        ----------------------------------------------------------------
        jobID:        jobID of the job record to modify
        fieldName:    public field name of the field
        increment:    increment is added to the current value of the field


self._logger = _LOGGER<EOL>assert (ClientJobsDAO._instance is None)<EOL>self.dbName = self._getDBName()<EOL>self._jobs = self._JobsTableInfo()<EOL>self._jobs.tableName = '<STR_LIT>' % (self.dbName)<EOL>self._models = self._ModelsTableInfo()<EOL>self._models.tableName = '<STR_LIT>' % (self.dbName)<EOL>self._connectionID = None<EOL>


self._logger.info('<STR_LIT>',<EOL>self.modelsTableName)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>' % (self.modelsTableName)<EOL>conn.cursor.execute(query)<EOL><DEDENT>


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetCompleted(self, jobID, completionReason, completionMsg,<EOL>useConnectionID = True):<DEDENT>

<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,<EOL>jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>result = conn.cursor.execute(query, sqlParams)<EOL>if result != <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (jobID))<EOL><DEDENT><DEDENT>

Change the status on the given job to completed

        Parameters:
        ----------------------------------------------------------------
        job:                 jobID of the job to mark as completed
        completionReason:    completionReason string
        completionMsg:       completionMsg string

        useConnectionID: True if the connection id of the calling function
        must be the same as the connection that created the job. Set
        to False for hypersearch workers


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelSetFields(self, modelID, fields, ignoreUnchanged = False):<DEDENT>

<EOL>assignmentExpressions = '<STR_LIT:U+002C>'.join(<EOL>'<STR_LIT>' % (self._models.pubToDBNameDict[f],) for f in fields.keys())<EOL>assignmentValues = list(fields.values())<EOL>query = '<STR_LIT>''<STR_LIT>'% (self.modelsTableName, assignmentExpressions)<EOL>sqlParams = assignmentValues + [modelID]<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>numAffectedRows = conn.cursor.execute(query, sqlParams)<EOL>self._logger.debug("<STR_LIT>",<EOL>numAffectedRows, query, sqlParams)<EOL><DEDENT>if numAffectedRows != <NUM_LIT:1> and not ignoreUnchanged:<EOL><INDENT>raise RuntimeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>") % (<EOL>fields, modelID, self._connectionID, numAffectedRows, query,<EOL>sqlParams,))<EOL><DEDENT>

Change the values of 1 or more fields in a model. Here, 'fields' is a
        dict with the name/value pairs to change. The names are the public names of
        the fields (camelBack, not the lower_case_only form as stored in the DB).

        Parameters:
        ----------------------------------------------------------------
        jobID:     jobID of the job record

        fields:    dictionary of fields to change

        ignoreUnchanged: The default behavior is to throw a
        RuntimeError if no rows are affected. This could either be
        because:
          1) Because there was no matching modelID
          2) or if the data to update matched the data in the DB exactly.

        Set this parameter to True if you expect case 2 and wish to
        supress the error.


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, [self.STATUS_COMPLETED])<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return tuple(r[<NUM_LIT:0>] for r in rows)<EOL>

Look through the jobs table and get the list of running jobs whose
        cancel field is true.

        Parameters:
        ----------------------------------------------------------------
        retval:      A (possibly empty) sequence of running job IDs with cancel field
                      set to true


self._logger.info('<STR_LIT>', jobID)<EOL>self.jobSetFields(jobID, {"<STR_LIT>" : True}, useConnectionID=False)<EOL>

Cancel the given job. This will update the cancel field in the
        jobs table and will result in the job being cancelled.

        Parameters:
        ----------------------------------------------------------------
        jobID:                 jobID of the job to mark as completed

        to False for hypersearch workers


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetFieldIfEqual(self, jobID, fieldName, newValue, curValue):<DEDENT>

<EOL>dbFieldName = self._jobs.pubToDBNameDict[fieldName]<EOL>conditionValue = []<EOL>if isinstance(curValue, bool):<EOL><INDENT>conditionExpression = '<STR_LIT>' % (<EOL>dbFieldName, {True:'<STR_LIT>', False:'<STR_LIT>'}[curValue])<EOL><DEDENT>elif curValue is None:<EOL><INDENT>conditionExpression = '<STR_LIT>' % (dbFieldName,)<EOL><DEDENT>else:<EOL><INDENT>conditionExpression = '<STR_LIT>' % (dbFieldName,)<EOL>conditionValue.append(curValue)<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, dbFieldName, conditionExpression)<EOL>sqlParams = [newValue, jobID] + conditionValue<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>result = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>return (result == <NUM_LIT:1>)<EOL>

Change the value of 1 field in a job to 'newValue', but only if the
        current value matches 'curValue'. The 'fieldName' is the public name of
        the field (camelBack, not the lower_case_only form as stored in the DB).

        This method is used for example by HypersearcWorkers to update the
        engWorkerState field periodically. By qualifying on curValue, it insures
        that only 1 worker at a time is elected to perform the next scheduled
        periodic sweep of the models.

        Parameters:
        ----------------------------------------------------------------
        jobID:        jobID of the job record to modify
        fieldName:    public field name of the field
        newValue:     new value of the field to set
        curValue:     current value to qualify against

        retval:       True if we successfully modified the field
                      False if curValue did not match


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetStatus(self, jobID, status, useConnectionID=True,):<DEDENT>

<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = [status, jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>result = conn.cursor.execute(query, sqlParams)<EOL>if result != <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>jobID, status))<EOL><DEDENT><DEDENT>

Change the status on the given job

        Parameters:
        ----------------------------------------------------------------
        job:        jobID of the job to change status
        status:     new status string (ClientJobsDAO.STATUS_xxxxx)

        useConnectionID: True if the connection id of the calling function
        must be the same as the connection that created the job. Set
        to False for hypersearch workers


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>' %  self.jobsTableName<EOL>conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])<EOL>activeJobCount = conn.cursor.fetchone()[<NUM_LIT:0>]<EOL><DEDENT>return activeJobCount<EOL>

Return the number of jobs for the given clientInfo and a status that is
        not completed.


@logExceptions(_LOGGER)<EOL><INDENT>def modelInsertAndStart(self, jobID, params, paramsHash, particleHash=None):<DEDENT>

<EOL>if particleHash is None:<EOL><INDENT>particleHash = paramsHash<EOL><DEDENT>paramsHash = self._normalizeHash(paramsHash)<EOL>particleHash = self._normalizeHash(particleHash)<EOL>def findExactMatchNoRetries(conn):<EOL><INDENT>return self._getOneMatchingRowNoRetries(<EOL>self._models, conn,<EOL>{'<STR_LIT>':jobID, '<STR_LIT>':paramsHash,<EOL>'<STR_LIT>':particleHash},<EOL>['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>@g_retrySQL<EOL>def findExactMatchWithRetries():<EOL><INDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>return findExactMatchNoRetries(conn)<EOL><DEDENT><DEDENT>row = findExactMatchWithRetries()<EOL>if row is not None:<EOL><INDENT>return (row[<NUM_LIT:0>], False)<EOL><DEDENT>@g_retrySQL<EOL>def insertModelWithRetries():<EOL><INDENT>"""<STR_LIT>"""<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = (jobID, params, self.STATUS_RUNNING, paramsHash,<EOL>particleHash, self._connectionID)<EOL>try:<EOL><INDENT>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>except Exception as e:<EOL><INDENT>if "<STR_LIT>" not in str(e):<EOL><INDENT>raise<EOL><DEDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, paramsHash.encode('<STR_LIT>'),<EOL>particleHash.encode('<STR_LIT>'), e)<EOL><DEDENT>else:<EOL><INDENT>if numRowsAffected == <NUM_LIT:1>:<EOL><INDENT>conn.cursor.execute('<STR_LIT>')<EOL>modelID = conn.cursor.fetchall()[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if modelID != <NUM_LIT:0>:<EOL><INDENT>return (modelID, True)<EOL><DEDENT>else:<EOL><INDENT>self._logger.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, paramsHash, particleHash)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._logger.error(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>numRowsAffected, jobID, paramsHash, particleHash)<EOL><DEDENT><DEDENT>row = findExactMatchNoRetries(conn)<EOL>if row is not None:<EOL><INDENT>(modelID, connectionID) = row<EOL>return (modelID, connectionID == self._connectionID)<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = [jobID, paramsHash, particleHash]<EOL>numRowsFound = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsFound == <NUM_LIT:1>, (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>') % (jobID, paramsHash, particleHash, numRowsFound)<EOL>(modelID,) = conn.cursor.fetchall()[<NUM_LIT:0>]<EOL>return (modelID, False)<EOL><DEDENT><DEDENT>return insertModelWithRetries()<EOL>

Insert a new unique model (based on params) into the model table in the
        "running" state. This will return two things: whether or not the model was
        actually inserted (i.e. that set of params isn't already in the table) and
        the modelID chosen for that set of params. Even if the model was not
        inserted by this call (it was already there) the modelID of the one already
        inserted is returned.

        Parameters:
        ----------------------------------------------------------------
        jobID:            jobID of the job to add models for
        params:           params for this model
        paramsHash        hash of the params, generated by the worker
        particleHash      hash of the particle info (for PSO). If not provided,
                          then paramsHash will be used.

        retval:           (modelID, wasInserted)
                          modelID: the model ID for this set of params
                          wasInserted: True if this call ended up inserting the
                          new model. False if this set of params was already in
                          the model table.


@logExceptions(_LOGGER)<EOL><INDENT>def modelsGetFields(self, modelIDs, fields):<DEDENT>

assert len(fields) >= <NUM_LIT:1>, '<STR_LIT>'<EOL>isSequence = isinstance(modelIDs, self._SEQUENCE_TYPES)<EOL>if isSequence:<EOL><INDENT>assert len(modelIDs) >=<NUM_LIT:1>, '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>modelIDs = [modelIDs]<EOL><DEDENT>rows = self._getMatchingRowsWithRetries(<EOL>self._models, dict(model_id=modelIDs),<EOL>['<STR_LIT>'] + [self._models.pubToDBNameDict[f] for f in fields])<EOL>if len(rows) < len(modelIDs):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (<EOL>(set(modelIDs) - set(r[<NUM_LIT:0>] for r in rows)),))<EOL><DEDENT>if not isSequence:<EOL><INDENT>return list(rows[<NUM_LIT:0>][<NUM_LIT:1>:])<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Fetch the values of 1 or more fields from a sequence of model records.
        Here, 'fields' is a list with the names of the fields to fetch. The names
        are the public names of the fields (camelBack, not the lower_case_only form
        as stored in the DB).

        WARNING!!!: The order of the results are NOT necessarily in the same order
        as the order of the model IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        modelIDs:      A single modelID or sequence of modelIDs
        fields:        A list  of fields to return

        Returns:  If modelIDs is a sequence:
                    a list of tuples->(modelID, [field1, field2,...])
                  If modelIDs is a single modelID:
                    a list of field values->[field1, field2,...]


def _insertOrGetUniqueJobNoRetries(<EOL>self, conn, client, cmdLine, jobHash, clientInfo, clientKey, params,<EOL>minimumWorkers, maximumWorkers, jobType, priority, alreadyRunning):

assert len(client) <= self.CLIENT_MAX_LEN, "<STR_LIT>" + repr(client)<EOL>assert cmdLine, "<STR_LIT>" + repr(cmdLine)<EOL>assert len(jobHash) == self.HASH_MAX_LEN, "<STR_LIT>" % len(jobHash)<EOL>if alreadyRunning:<EOL><INDENT>initStatus = self.STATUS_TESTMODE<EOL><DEDENT>else:<EOL><INDENT>initStatus = self.STATUS_NOTSTARTED<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = (initStatus, client, clientInfo, clientKey, cmdLine, params,<EOL>jobHash, minimumWorkers, maximumWorkers, priority, jobType)<EOL>numRowsInserted = conn.cursor.execute(query, sqlParams)<EOL>jobID = <NUM_LIT:0><EOL>if numRowsInserted == <NUM_LIT:1>:<EOL><INDENT>conn.cursor.execute('<STR_LIT>')<EOL>jobID = conn.cursor.fetchall()[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if jobID == <NUM_LIT:0>:<EOL><INDENT>self._logger.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobType, client, _abbreviate(clientInfo, <NUM_LIT:32>), clientKey, jobHash,<EOL>cmdLine)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert numRowsInserted == <NUM_LIT:0>, repr(numRowsInserted)<EOL><DEDENT>if jobID == <NUM_LIT:0>:<EOL><INDENT>row = self._getOneMatchingRowNoRetries(<EOL>self._jobs, conn, dict(client=client, job_hash=jobHash), ['<STR_LIT>'])<EOL>assert row is not None<EOL>assert len(row) == <NUM_LIT:1>, '<STR_LIT>' + repr(len(row))<EOL>jobID = row[<NUM_LIT:0>]<EOL><DEDENT>if alreadyRunning:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, (self._connectionID, jobID))<EOL><DEDENT>return jobID<EOL>

Attempt to insert a row with the given parameters into the jobs table.
        Return jobID of the inserted row, or of an existing row with matching
        client/jobHash key.

        The combination of client and jobHash are expected to be unique (enforced
        by a unique index on the two columns).

        NOTE: It's possibe that this or another process (on this or another machine)
         already inserted a row with matching client/jobHash key (e.g.,
         StreamMgr). This may also happen undetected by this function due to a
         partially-successful insert operation (e.g., row inserted, but then
         connection was lost while reading response) followed by retries either of
         this function or in SteadyDB module.

        Parameters:
        ----------------------------------------------------------------
        conn:            Owned connection acquired from ConnectionFactory.get()
        client:          Name of the client submitting the job
        cmdLine:         Command line to use to launch each worker process; must be
                          a non-empty string
        jobHash:         unique hash of this job. The caller must insure that this,
                          together with client, uniquely identifies this job request
                          for the purposes of detecting duplicates.
        clientInfo:      JSON encoded dict of client specific information.
        clientKey:       Foreign key.
        params:          JSON encoded dict of the parameters for the job. This
                          can be fetched out of the database by the worker processes
                          based on the jobID.
        minimumWorkers:  minimum number of workers design at a time.
        maximumWorkers:  maximum number of workers desired at a time.
        priority:        Job scheduling priority; 0 is the default priority (
                          ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are
                          higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),
                          and negative values are lower priority (down to
                          ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will
                          be scheduled to run at the expense of the lower-priority
                          jobs, and higher-priority job tasks will preempt those
                          with lower priority if there is inadequate supply of
                          scheduling slots. Excess lower priority job tasks will
                          starve as long as slot demand exceeds supply. Most jobs
                          should be scheduled with DEFAULT_JOB_PRIORITY. System jobs
                          that must run at all cost, such as Multi-Model-Master,
                          should be scheduled with MAX_JOB_PRIORITY.
        alreadyRunning:  Used for unit test purposes only. This inserts the job
                          in the running state. It is used when running a worker
                          in standalone mode without hadoop- it gives it a job
                          record to work with.

        retval:           jobID of the inserted jobs row, or of an existing jobs row
                           with matching client/jobHash key


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = [self.STATUS_RUNNING, self._connectionID,<EOL>jobID, self.STATUS_NOTSTARTED]<EOL>numRowsUpdated = conn.cursor.execute(query, sqlParams)<EOL>if numRowsUpdated != <NUM_LIT:1>:<EOL><INDENT>self._logger.warn('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', numRowsUpdated)<EOL><DEDENT><DEDENT>return<EOL>

Place the given job in STATUS_RUNNING mode; the job is expected to be
        STATUS_NOTSTARTED.

        NOTE: this function was factored out of jobStartNext because it's also
         needed for testing (e.g., test_client_jobs_dao.py)


<EOL>combinedResults = None<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT:U+0020>'.join([<EOL>'<STR_LIT>' % (self.jobsTableName, self.modelsTableName),<EOL>'<STR_LIT>' % self.jobsTableName,<EOL>'<STR_LIT>' % self.modelsTableName,<EOL>'<STR_LIT>'])<EOL>conn.cursor.execute(query, (jobID,))<EOL>if conn.cursor.rowcount > <NUM_LIT:0>:<EOL><INDENT>combinedResults = [<EOL>ClientJobsDAO._combineResults(<EOL>result, self._jobs.jobInfoNamedTuple,<EOL>self._models.modelInfoNamedTuple<EOL>) for result in conn.cursor.fetchall()]<EOL><DEDENT><DEDENT>if combinedResults is not None:<EOL><INDENT>return combinedResults<EOL><DEDENT>raise RuntimeError("<STR_LIT>" % (jobID))<EOL>

Get all info about a job, with model details, if available.

        Parameters:
        ----------------------------------------------------------------
        job:    jobID of the job to query
        retval: A sequence of two-tuples if the jobID exists in the jobs
                 table (exeption is raised if it doesn't exist). Each two-tuple
                 contains an instance of jobInfoNamedTuple as the first element and
                 an instance of modelInfoNamedTuple as the second element. NOTE: In
                 the case where there are no matching model rows, a sequence of one
                 two-tuple will still be returned, but the modelInfoNamedTuple
                 fields will be None, and the jobInfoNamedTuple fields will be
                 populated.


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelUpdateResults(self, modelID, results=None, metricValue =None,<EOL>numRecords=None):<DEDENT>

assignmentExpressions = ['<STR_LIT>',<EOL>'<STR_LIT>']<EOL>assignmentValues = []<EOL>if results is not None:<EOL><INDENT>assignmentExpressions.append('<STR_LIT>')<EOL>assignmentValues.append(results)<EOL><DEDENT>if numRecords is not None:<EOL><INDENT>assignmentExpressions.append('<STR_LIT>')<EOL>assignmentValues.append(numRecords)<EOL><DEDENT>if metricValue is not None and (metricValue==metricValue):<EOL><INDENT>assignmentExpressions.append('<STR_LIT>')<EOL>assignmentValues.append(float(metricValue))<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>'% (self.modelsTableName, '<STR_LIT:U+002C>'.join(assignmentExpressions))<EOL>sqlParams = assignmentValues + [modelID, self._connectionID]<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if numRowsAffected != <NUM_LIT:1>:<EOL><INDENT>raise InvalidConnectionException(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % (modelID,self._connectionID, numRowsAffected,))<EOL><DEDENT>

Update the results string, and/or num_records fields of
        a model. This will fail if the model does not currently belong to this
        client (connection_id doesn't match).

        Parameters:
        ----------------------------------------------------------------
        modelID:      model ID of model to modify
        results:      new results, or None to ignore
        metricValue:  the value of the metric being optimized, or None to ignore
        numRecords:   new numRecords, or None to ignore


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelsGetFieldsForJob(self, jobID, fields, ignoreKilled=False):<DEDENT>

assert len(fields) >= <NUM_LIT:1>, '<STR_LIT>'<EOL>dbFields = [self._models.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(dbFields)<EOL>query = '<STR_LIT>''<STR_LIT>'% (dbFieldsStr, self.modelsTableName)<EOL>sqlParams = [jobID]<EOL>if ignoreKilled:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self.CMPL_REASON_KILLED)<EOL><DEDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>conn.cursor.execute(query, sqlParams)<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>if rows is None:<EOL><INDENT>self._logger.error("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>query, traceback.format_exc())<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Gets the specified fields for all the models for a single job. This is
        similar to modelsGetFields

        Parameters:
        ----------------------------------------------------------------
        jobID:              jobID for the models to be searched
        fields:             A list  of fields to return
        ignoreKilled:       (True/False). If True, this will ignore models that
                            have been killed

        Returns: a (possibly empty) list of tuples as follows
          [
            (model_id1, [field1, ..., fieldn]),
            (model_id2, [field1, ..., fieldn]),
            (model_id3, [field1, ..., fieldn])
                        ...
          ]

        NOTE: since there is a window of time between a job getting inserted into
         jobs table and the job's worker(s) starting up and creating models, an
         empty-list result is one of the normal outcomes.


if text is not None and len(text) > threshold:<EOL><INDENT>text = text[:threshold] + "<STR_LIT>"<EOL><DEDENT>return text<EOL>

Abbreviate the given text to threshold chars and append an ellipsis if its
    length exceeds threshold; used for logging;

    NOTE: the resulting text could be longer than threshold due to the ellipsis


@logExceptions(_LOGGER)<EOL><INDENT>def jobsGetFields(self, jobIDs, fields, requireAll=True):<DEDENT>

assert isinstance(jobIDs, self._SEQUENCE_TYPES)<EOL>assert len(jobIDs) >=<NUM_LIT:1><EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._jobs, dict(job_id=jobIDs),<EOL>['<STR_LIT>'] + [self._jobs.pubToDBNameDict[x] for x in fields])<EOL>if requireAll and len(rows) < len(jobIDs):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (<EOL>(set(jobIDs) - set(r[<NUM_LIT:0>] for r in rows)),))<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Fetch the values of 1 or more fields from a sequence of job records.
        Here, 'fields' is a sequence (list or tuple) with the names of the fields to
        fetch. The names are the public names of the fields (camelBack, not the
        lower_case_only form as stored in the DB).

        WARNING!!!: The order of the results are NOT necessarily in the same order as
        the order of the job IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        jobIDs:        A sequence of jobIDs
        fields:        A list  of fields to return for each jobID

        Returns:      A list of tuples->(jobID, [field1, field2,...])


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelSetCompleted(self, modelID, completionReason, completionMsg,<EOL>cpuTime=<NUM_LIT:0>, useConnectionID=True):<DEDENT>

if completionMsg is None:<EOL><INDENT>completionMsg = '<STR_LIT>'<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,<EOL>cpuTime, modelID]<EOL>if useConnectionID:<EOL><INDENT>query += "<STR_LIT>"<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if numRowsAffected != <NUM_LIT:1>:<EOL><INDENT>raise InvalidConnectionException(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % (modelID, self._connectionID, numRowsAffected))<EOL><DEDENT>

Mark a model as completed, with the given completionReason and
        completionMsg. This will fail if the model does not currently belong to this
        client (connection_id doesn't match).

        Parameters:
        ----------------------------------------------------------------
        modelID:             model ID of model to modify
        completionReason:    completionReason string
        completionMsg:       completionMsg string
        cpuTime:             amount of CPU time spent on this model
        useConnectionID:     True if the connection id of the calling function
                              must be the same as the connection that created the
                              job. Set to True for hypersearch workers, which use
                              this mechanism for orphaned model detection.


row = self._getOneMatchingRowWithRetries(<EOL>self._jobs, dict(job_id=jobID),<EOL>[self._jobs.pubToDBNameDict[n]<EOL>for n in self._jobs.jobInfoNamedTuple._fields])<EOL>if row is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (jobID))<EOL><DEDENT>return self._jobs.jobInfoNamedTuple._make(row)<EOL>

Get all info about a job

        Parameters:
        ----------------------------------------------------------------
        job:    jobID of the job to query
        retval:  namedtuple containing the job info.


def _getOneMatchingRowNoRetries(self, tableInfo, conn, fieldsToMatch,<EOL>selectFieldNames):

rows = self._getMatchingRowsNoRetries(tableInfo, conn, fieldsToMatch,<EOL>selectFieldNames, maxRows=<NUM_LIT:1>)<EOL>if rows:<EOL><INDENT>assert len(rows) == <NUM_LIT:1>, repr(len(rows))<EOL>result = rows[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>result = None<EOL><DEDENT>return result<EOL>

Return a single matching row with the requested field values from the
        the requested table or None if nothing matched.

        tableInfo:       Table information: a ClientJobsDAO._TableInfoBase  instance
        conn:            Owned connection acquired from ConnectionFactory.get()
        fieldsToMatch:   Dictionary of internal fieldName/value mappings that
                         identify the desired rows. If a value is an instance of
                         ClientJobsDAO._SEQUENCE_TYPES (list/set/tuple), then the
                         operator 'IN' will be used in the corresponding SQL
                         predicate; if the value is bool: "IS TRUE/FALSE"; if the
                         value is None: "IS NULL"; '=' will be used for all other
                         cases.
        selectFieldNames:
                         list of fields to return, using internal field names

        retval:          A sequence of field values of the matching row in the order
                          of the given field names; or None if there was no match.


return self._connectionID<EOL>

Return our connection ID. This can be used for worker identification
        purposes.

        NOTE: the actual MySQL connection ID used in queries may change from time
         to time if connection is re-acquired (e.g., upon MySQL server restart) or
         when more than one entry from the connection pool has been used (e.g.,
         multi-threaded apps)


@g_retrySQL<EOL><INDENT>def _getOneMatchingRowWithRetries(self, tableInfo, fieldsToMatch,<EOL>selectFieldNames):<DEDENT>

with ConnectionFactory.get() as conn:<EOL><INDENT>return self._getOneMatchingRowNoRetries(tableInfo, conn, fieldsToMatch,<EOL>selectFieldNames)<EOL><DEDENT>

Like _getOneMatchingRowNoRetries(), but with retries on transient MySQL
        failures


def _resumeJobNoRetries(self, conn, jobID, alreadyRunning):

<EOL>if alreadyRunning:<EOL><INDENT>initStatus = self.STATUS_TESTMODE<EOL><DEDENT>else:<EOL><INDENT>initStatus = self.STATUS_NOTSTARTED<EOL><DEDENT>assignments = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>]<EOL>assignmentValues = [initStatus]<EOL>if alreadyRunning:<EOL><INDENT>assignments += ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>assignmentValues.append(self._connectionID)<EOL><DEDENT>else:<EOL><INDENT>assignments += ['<STR_LIT>', '<STR_LIT>']<EOL><DEDENT>assignments = '<STR_LIT:U+002CU+0020>'.join(assignments)<EOL>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, assignments)<EOL>sqlParams = assignmentValues + [jobID, self.STATUS_COMPLETED]<EOL>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsAffected <= <NUM_LIT:1>, repr(numRowsAffected)<EOL>if numRowsAffected == <NUM_LIT:0>:<EOL><INDENT>self._logger.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", jobID)<EOL><DEDENT>return<EOL>

Resumes processing of an existing job that is presently in the
        STATUS_COMPLETED state.

        NOTE: this is primarily for resuming suspended Production and Stream Jobs; DO
         NOT use it on Hypersearch jobs.

        This prepares an existing job entry to resume processing. The CJM is always
        periodically sweeping the jobs table and when it finds a job that is ready
        to run, it will proceed to start it up on Hadoop.

        Parameters:
        ----------------------------------------------------------------
        conn:            Owned connection acquired from ConnectionFactory.get()
        jobID:          jobID of the job to resume
        alreadyRunning: Used for unit test purposes only. This inserts the job
                         in the running state. It is used when running a worker
                         in standalone mode without hadoop.

        raises:         Throws a RuntimeError if no rows are affected. This could
                        either be because:
                          1) Because there was not matching jobID
                          2) or if the status of the job was not STATUS_COMPLETED.

        retval:            nothing


@logExceptions(_LOGGER)<EOL><INDENT>def _initTables(self, cursor, deleteOldVersions, recreate):<DEDENT>

<EOL>if deleteOldVersions:<EOL><INDENT>self._logger.info(<EOL>"<STR_LIT>",<EOL>traceback.format_stack())<EOL>for i in range(self._DB_VERSION):<EOL><INDENT>cursor.execute('<STR_LIT>' %<EOL>(self.__getDBNameForVersion(i),))<EOL><DEDENT><DEDENT>if recreate:<EOL><INDENT>self._logger.info(<EOL>"<STR_LIT>",<EOL>self.dbName, traceback.format_stack())<EOL>cursor.execute('<STR_LIT>' % (self.dbName))<EOL><DEDENT>cursor.execute('<STR_LIT>' % (self.dbName))<EOL>cursor.execute('<STR_LIT>' % (self.dbName))<EOL>output = cursor.fetchall()<EOL>tableNames = [x[<NUM_LIT:0>] for x in output]<EOL>if '<STR_LIT>' not in tableNames:<EOL><INDENT>self._logger.info("<STR_LIT>", self.jobsTableName)<EOL>fields = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % (self.CLIENT_MAX_LEN),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % (self.HASH_MAX_LEN),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'  %self.CMPL_REASON_SUCCESS,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % self.DEFAULT_JOB_PRIORITY,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'  %self.CLEAN_NOT_DONE,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'<EOL>]<EOL>options = [<EOL>'<STR_LIT>',<EOL>]<EOL>query = '<STR_LIT>' %(self.jobsTableName, '<STR_LIT:U+002C>'.join(fields), '<STR_LIT:U+002C>'.join(options))<EOL>cursor.execute(query)<EOL><DEDENT>if '<STR_LIT>' not in tableNames:<EOL><INDENT>self._logger.info("<STR_LIT>", self.modelsTableName)<EOL>fields = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % (self.HASH_MAX_LEN),<EOL>'<STR_LIT>' % (self.HASH_MAX_LEN),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>]<EOL>options = [<EOL>'<STR_LIT>',<EOL>]<EOL>query = '<STR_LIT>' %(self.modelsTableName, '<STR_LIT:U+002C>'.join(fields), '<STR_LIT:U+002C>'.join(options))<EOL>cursor.execute(query)<EOL><DEDENT>cursor.execute('<STR_LIT>' % (self.jobsTableName))<EOL>fields = cursor.fetchall()<EOL>self._jobs.dbFieldNames = [str(field[<NUM_LIT:0>]) for field in fields]<EOL>cursor.execute('<STR_LIT>' % (self.modelsTableName))<EOL>fields = cursor.fetchall()<EOL>self._models.dbFieldNames = [str(field[<NUM_LIT:0>]) for field in fields]<EOL>self._jobs.publicFieldNames = [self._columnNameDBToPublic(x)<EOL>for x in self._jobs.dbFieldNames]<EOL>self._models.publicFieldNames = [self._columnNameDBToPublic(x)<EOL>for x in self._models.dbFieldNames]<EOL>self._jobs.pubToDBNameDict = dict(<EOL>list(zip(self._jobs.publicFieldNames, self._jobs.dbFieldNames)))<EOL>self._jobs.dbToPubNameDict = dict(<EOL>list(zip(self._jobs.dbFieldNames, self._jobs.publicFieldNames)))<EOL>self._models.pubToDBNameDict = dict(<EOL>list(zip(self._models.publicFieldNames, self._models.dbFieldNames)))<EOL>self._models.dbToPubNameDict = dict(<EOL>list(zip(self._models.dbFieldNames, self._models.publicFieldNames)))<EOL>self._models.modelInfoNamedTuple = collections.namedtuple(<EOL>'<STR_LIT>', self._models.publicFieldNames)<EOL>self._jobs.jobInfoNamedTuple = collections.namedtuple(<EOL>'<STR_LIT>', self._jobs.publicFieldNames)<EOL>return<EOL>

Initialize tables, if needed

        Parameters:
        ----------------------------------------------------------------
        cursor:              SQL cursor
        deleteOldVersions:   if true, delete any old versions of the DB left
                              on the server
        recreate:            if true, recreate the database from scratch even
                              if it already exists.


<EOL>dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(['<STR_LIT>'] + dbFields)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>' % (dbFieldsStr, self.jobsTableName)<EOL>conn.cursor.execute(query)<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return rows<EOL>

Fetch jobIDs for jobs in the table with optional fields


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def connect(self, deleteOldVersions=False, recreate=False):<DEDENT>

<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>self._initTables(cursor=conn.cursor, deleteOldVersions=deleteOldVersions,<EOL>recreate=recreate)<EOL>conn.cursor.execute('<STR_LIT>')<EOL>self._connectionID = conn.cursor.fetchall()[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>self._logger.info("<STR_LIT>", self._connectionID)<EOL><DEDENT>return<EOL>

Locate the current version of the jobs DB or create a new one, and
        optionally delete old versions laying around. If desired, this method
        can be called at any time to re-create the tables from scratch, delete
        old versions of the database, etc.

        Parameters:
        ----------------------------------------------------------------
        deleteOldVersions:   if true, delete any old versions of the DB left
                              on the server
        recreate:            if true, recreate the database from scratch even
                              if it already exists.


rows = self._getMatchingRowsWithRetries(self._models, dict(job_id=jobID),<EOL>['<STR_LIT>'])<EOL>return [r[<NUM_LIT:0>] for r in rows]<EOL>

Fetch all the modelIDs that correspond to a given jobID; empty sequence
        if none


assert isinstance(modelIDs, self._SEQUENCE_TYPES), (<EOL>"<STR_LIT>") % (type(modelIDs),)<EOL>assert modelIDs, "<STR_LIT>"<EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._models, dict(model_id=modelIDs),<EOL>[self._models.pubToDBNameDict[f]<EOL>for f in self._models.modelInfoNamedTuple._fields])<EOL>results = [self._models.modelInfoNamedTuple._make(r) for r in rows]<EOL>assert len(results) == len(modelIDs), "<STR_LIT>" % (<EOL>set(modelIDs) - set(r.modelId for r in results))<EOL>return results<EOL>

Get ALL info for a set of models

        WARNING!!!: The order of the results are NOT necessarily in the same order as
        the order of the model IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        modelIDs:    list of model IDs
        retval:      list of nametuples containing all the fields stored for each
                        model.


assert isinstance(modelIDs, self._SEQUENCE_TYPES), (<EOL>"<STR_LIT>") % type(modelIDs)<EOL>assert len(modelIDs) >= <NUM_LIT:1>, "<STR_LIT>"<EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._models, {'<STR_LIT>' : modelIDs},<EOL>[self._models.pubToDBNameDict[f]<EOL>for f in self._models.getResultAndStatusNamedTuple._fields])<EOL>assert len(rows) == len(modelIDs), "<STR_LIT>" % (<EOL>(set(modelIDs) - set(r[<NUM_LIT:0>] for r in rows)),)<EOL>return [self._models.getResultAndStatusNamedTuple._make(r) for r in rows]<EOL>

Get the results string and other status fields for a set of models.

        WARNING!!!: The order of the results are NOT necessarily in the same order
        as the order of the model IDs passed in!!!

        For each model, this returns a tuple containing:
         (modelID, results, status, updateCounter, numRecords, completionReason,
             completionMsg, engParamsHash

        Parameters:
        ----------------------------------------------------------------
        modelIDs:    list of model IDs
        retval:      list of result tuples. Each tuple contains:
                        (modelID, results, status, updateCounter, numRecords,
                          completionReason, completionMsg, engParamsHash)


<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, [self._connectionID, self.STATUS_RUNNING])<EOL><DEDENT>return<EOL>

Look through the jobs table and reactivate all that are already in the
        running state by setting their _eng_allocate_new_workers fields to True;
        used by Nupic Scheduler as part of its failure-recovery procedure.


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>' %  self.jobsTableName<EOL>conn.cursor.execute(query, [clientKey, self.STATUS_COMPLETED])<EOL>activeJobCount = conn.cursor.fetchone()[<NUM_LIT:0>]<EOL><DEDENT>return activeJobCount<EOL>

Return the number of jobs for the given clientKey and a status that is
        not completed.


@logExceptions(_LOGGER)<EOL><INDENT>def jobInsertUnique(self, client, cmdLine, jobHash, clientInfo='<STR_LIT>',<EOL>clientKey='<STR_LIT>', params='<STR_LIT>', minimumWorkers=<NUM_LIT:0>,<EOL>maximumWorkers=<NUM_LIT:0>, jobType='<STR_LIT>',<EOL>priority=DEFAULT_JOB_PRIORITY):<DEDENT>

assert cmdLine, "<STR_LIT>" + repr(cmdLine)<EOL>@g_retrySQL<EOL>def insertUniqueWithRetries():<EOL><INDENT>jobHashValue = self._normalizeHash(jobHash)<EOL>jobID = None<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>row = self._getOneMatchingRowNoRetries(<EOL>self._jobs, conn, dict(client=client, job_hash=jobHashValue),<EOL>['<STR_LIT>', '<STR_LIT:status>'])<EOL>if row is not None:<EOL><INDENT>(jobID, status) = row<EOL>if status == self.STATUS_COMPLETED:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = (clientInfo, clientKey, cmdLine, params,<EOL>minimumWorkers, maximumWorkers, priority,<EOL>jobType, jobID, self.STATUS_COMPLETED)<EOL>numRowsUpdated = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsUpdated <= <NUM_LIT:1>, repr(numRowsUpdated)<EOL>if numRowsUpdated == <NUM_LIT:0>:<EOL><INDENT>self._logger.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", jobID)<EOL><DEDENT>self._resumeJobNoRetries(conn, jobID, alreadyRunning=False)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>jobID = self._insertOrGetUniqueJobNoRetries(<EOL>conn, client=client, cmdLine=cmdLine, jobHash=jobHashValue,<EOL>clientInfo=clientInfo, clientKey=clientKey, params=params,<EOL>minimumWorkers=minimumWorkers, maximumWorkers=maximumWorkers,<EOL>jobType=jobType, priority=priority, alreadyRunning=False)<EOL><DEDENT>return jobID<EOL><DEDENT><DEDENT>try:<EOL><INDENT>jobID = insertUniqueWithRetries()<EOL><DEDENT>except:<EOL><INDENT>self._logger.exception(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobType, client, _abbreviate(clientInfo, <NUM_LIT>), clientKey, jobHash,<EOL>cmdLine)<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>self._logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, jobType, client, _abbreviate(clientInfo, <NUM_LIT>), clientKey,<EOL>jobHash, cmdLine)<EOL><DEDENT>return jobID<EOL>

Add an entry to the jobs table for a new job request, but only if the
        same job, by the same client is not already running. If the job is already
        running, or queued up to run, this call does nothing. If the job does not
        exist in the jobs table or has completed, it will be inserted and/or started
        up again.

        This method is called by clients, like StreamMgr, that wish to only start up
        a job if it hasn't already been started up.

        Parameters:
        ----------------------------------------------------------------
        client:          Name of the client submitting the job
        cmdLine:         Command line to use to launch each worker process; must be
                          a non-empty string
        jobHash:         unique hash of this job. The client must insure that this
                          uniquely identifies this job request for the purposes
                          of detecting duplicates.
        clientInfo:      JSON encoded dict of client specific information.
        clientKey:       Foreign key.
        params:          JSON encoded dict of the parameters for the job. This
                          can be fetched out of the database by the worker processes
                          based on the jobID.
        minimumWorkers:  minimum number of workers design at a time.
        maximumWorkers:  maximum number of workers desired at a time.
        jobType:         The type of job that this is. This should be one of the
                          JOB_TYPE_XXXX enums. This is needed to allow a standard
                          way of recognizing a job's function and capabilities.
        priority:        Job scheduling priority; 0 is the default priority (
                          ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are
                          higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),
                          and negative values are lower priority (down to
                          ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will
                          be scheduled to run at the expense of the lower-priority
                          jobs, and higher-priority job tasks will preempt those
                          with lower priority if there is inadequate supply of
                          scheduling slots. Excess lower priority job tasks will
                          starve as long as slot demand exceeds supply. Most jobs
                          should be scheduled with DEFAULT_JOB_PRIORITY. System jobs
                          that must run at all cost, such as Multi-Model-Master,
                          should be scheduled with MAX_JOB_PRIORITY.

        retval:          jobID of the newly inserted or existing job.


if "<STR_LIT>" not in distributionParams or "<STR_LIT>" not in distributionParams:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if x < distributionParams["<STR_LIT>"]:<EOL><INDENT>xp = <NUM_LIT:2> * distributionParams["<STR_LIT>"] - x<EOL>return tailProbability(xp, distributionParams)<EOL><DEDENT>z = (x - distributionParams["<STR_LIT>"]) / distributionParams["<STR_LIT>"]<EOL>return <NUM_LIT:0.5> * math.erfc(z/<NUM_LIT>)<EOL>

Given the normal distribution specified by the mean and standard deviation
in distributionParams, return the probability of getting samples further
from the mean. For values above the mean, this is the probability of getting
samples > x and for values below the mean, the probability of getting
samples < x. This is the Q-function: the tail probability of the normal distribution.

:param distributionParams: dict with 'mean' and 'stdev' of the distribution


<EOL>return math.log(<NUM_LIT> - likelihood) / -<NUM_LIT><EOL>

Compute a log scale representation of the likelihood value. Since the
likelihood computations return low probabilities that often go into four 9's
or five 9's, a log value is more useful for visualization, thresholding,
etc.


params = {<EOL>"<STR_LIT:name>": "<STR_LIT>",<EOL>"<STR_LIT>": numpy.mean(sampleData),<EOL>"<STR_LIT>": numpy.var(sampleData),<EOL>}<EOL>if performLowerBoundCheck:<EOL><INDENT>if params["<STR_LIT>"] < <NUM_LIT>:<EOL><INDENT>params["<STR_LIT>"] = <NUM_LIT><EOL><DEDENT>if params["<STR_LIT>"] < <NUM_LIT>:<EOL><INDENT>params["<STR_LIT>"] = <NUM_LIT><EOL><DEDENT><DEDENT>if params["<STR_LIT>"] > <NUM_LIT:0>:<EOL><INDENT>params["<STR_LIT>"] = math.sqrt(params["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>params["<STR_LIT>"] = <NUM_LIT:0><EOL><DEDENT>return params<EOL>

:param sampleData:
:type sampleData: Numpy array.
:param performLowerBoundCheck:
:type performLowerBoundCheck: bool
:returns: A dict containing the parameters of a normal distribution based on
    the ``sampleData``.


if verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>", len(anomalyScores))<EOL>print("<STR_LIT>", skipRecords)<EOL>print("<STR_LIT>", anomalyScores[<NUM_LIT:0>:min(<NUM_LIT:20>, len(anomalyScores))])<EOL><DEDENT>if len(anomalyScores) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>aggRecordList, historicalValues, total =  _anomalyScoreMovingAverage(<EOL>anomalyScores,<EOL>windowSize = averagingWindow,<EOL>verbosity = verbosity)<EOL>s = [r[<NUM_LIT:2>] for r in aggRecordList]<EOL>dataValues = numpy.array(s)<EOL>if len(aggRecordList) <= skipRecords:<EOL><INDENT>distributionParams = nullDistribution(verbosity = verbosity)<EOL><DEDENT>else:<EOL><INDENT>distributionParams = estimateNormal(dataValues[skipRecords:])<EOL>s = [r[<NUM_LIT:1>] for r in aggRecordList]<EOL>if all([isinstance(r[<NUM_LIT:1>], numbers.Number) for r in aggRecordList]):<EOL><INDENT>metricValues = numpy.array(s)<EOL>metricDistribution = estimateNormal(metricValues[skipRecords:],<EOL>performLowerBoundCheck=False)<EOL>if metricDistribution["<STR_LIT>"] < <NUM_LIT>:<EOL><INDENT>distributionParams = nullDistribution(verbosity = verbosity)<EOL><DEDENT><DEDENT><DEDENT>likelihoods = numpy.array(dataValues, dtype=float)<EOL>for i, s in enumerate(dataValues):<EOL><INDENT>likelihoods[i] = tailProbability(s, distributionParams)<EOL><DEDENT>filteredLikelihoods = numpy.array(<EOL>_filterLikelihoods(likelihoods) )<EOL>params = {<EOL>"<STR_LIT>":       distributionParams,<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": historicalValues,<EOL>"<STR_LIT>":            total,<EOL>"<STR_LIT>":       averagingWindow,<EOL>},<EOL>"<STR_LIT>":<EOL>list(likelihoods[-min(averagingWindow, len(likelihoods)):]),<EOL>}<EOL>if verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>print(params)<EOL>print("<STR_LIT>", len(likelihoods))<EOL>print("<STR_LIT>", (<EOL>filteredLikelihoods[<NUM_LIT:0>:min(<NUM_LIT:20>, len(filteredLikelihoods))] ))<EOL>print("<STR_LIT>")<EOL><DEDENT>return (filteredLikelihoods, aggRecordList, params)<EOL>

Given a series of anomaly scores, compute the likelihood for each score. This
function should be called once on a bunch of historical anomaly scores for an
initial estimate of the distribution. It should be called again every so often
(say every 50 records) to update the estimate.

:param anomalyScores: a list of records. Each record is a list with the
                      following three elements: [timestamp, value, score]

                      Example::

                          [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]

                      For best results, the list should be between 1000
                      and 10,000 records
:param averagingWindow: integer number of records to average over
:param skipRecords: integer specifying number of records to skip when
                    estimating distributions. If skip records are >=
                    len(anomalyScores), a very broad distribution is returned
                    that makes everything pretty likely.
:param verbosity: integer controlling extent of printouts for debugging

                    0 = none
                    1 = occasional information
                    2 = print every record

:returns: 3-tuple consisting of:

          - likelihoods

            numpy array of likelihoods, one for each aggregated point

          - avgRecordList

            list of averaged input records

          - params

            a small JSON dict that contains the state of the estimator


def anomalyProbability(self, value, anomalyScore, timestamp=None):

if timestamp is None:<EOL><INDENT>timestamp = self._iteration<EOL><DEDENT>dataPoint = (timestamp, value, anomalyScore)<EOL>if self._iteration < self._probationaryPeriod:<EOL><INDENT>likelihood = <NUM_LIT:0.5><EOL><DEDENT>else:<EOL><INDENT>if ( (self._distribution is None) or<EOL>(self._iteration % self._reestimationPeriod == <NUM_LIT:0>) ):<EOL><INDENT>numSkipRecords = self._calcSkipRecords(<EOL>numIngested=self._iteration,<EOL>windowSize=self._historicalScores.maxlen,<EOL>learningPeriod=self._learningPeriod)<EOL>_, _, self._distribution = estimateAnomalyLikelihoods(<EOL>self._historicalScores,<EOL>skipRecords=numSkipRecords)<EOL><DEDENT>likelihoods, _, self._distribution = updateAnomalyLikelihoods(<EOL>[dataPoint],<EOL>self._distribution)<EOL>likelihood = <NUM_LIT:1.0> - likelihoods[<NUM_LIT:0>]<EOL><DEDENT>self._historicalScores.append(dataPoint)<EOL>self._iteration += <NUM_LIT:1><EOL>return likelihood<EOL>

Compute the probability that the current value plus anomaly score represents
an anomaly given the historical distribution of anomaly scores. The closer
the number is to 1, the higher the chance it is an anomaly.

:param value: the current metric ("raw") input value, eg. "orange", or
               '21.2' (deg. Celsius), ...
:param anomalyScore: the current anomaly score
:param timestamp: [optional] timestamp of the ocurrence,
                   default (None) results in using iteration step.
:returns: the anomalyLikelihood for this record.


redThreshold    = <NUM_LIT:1.0> - redThreshold<EOL>yellowThreshold = <NUM_LIT:1.0> - yellowThreshold<EOL>filteredLikelihoods = [likelihoods[<NUM_LIT:0>]]<EOL>for i, v in enumerate(likelihoods[<NUM_LIT:1>:]):<EOL><INDENT>if v <= redThreshold:<EOL><INDENT>if likelihoods[i] > redThreshold:<EOL><INDENT>filteredLikelihoods.append(v)<EOL><DEDENT>else:<EOL><INDENT>filteredLikelihoods.append(yellowThreshold)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>filteredLikelihoods.append(v)<EOL><DEDENT><DEDENT>return filteredLikelihoods<EOL>

Filter the list of raw (pre-filtered) likelihoods so that we only preserve
sharp increases in likelihood. 'likelihoods' can be a numpy array of floats or
a list of floats.

:returns: A new list of floats likelihoods containing the filtered values.


def generateSimpleCoincMatrix(nCoinc=<NUM_LIT:10>, length=<NUM_LIT>, activity=<NUM_LIT:50>):

assert nCoinc*activity<=length, "<STR_LIT>"<EOL>coincMatrix = SM32(<NUM_LIT:0>, length)<EOL>coinc = numpy.zeros(length, dtype='<STR_LIT>')<EOL>for i in range(nCoinc):<EOL><INDENT>coinc[:] = <NUM_LIT:0><EOL>coinc[i*activity:(i+<NUM_LIT:1>)*activity] = <NUM_LIT:1><EOL>coincMatrix.addRow(coinc)<EOL><DEDENT>return coincMatrix<EOL>

Generate a non overlapping coincidence matrix. This is used to generate random
inputs to the temporal learner and to compare the predicted output against.

It generates a matrix of nCoinc rows, each row has length 'length' and has
a total of 'activity' bits on.

Parameters:
-----------------------------------------------
nCoinc:        the number of rows to generate
length:        the length of each row
activity:      the number of ones to put into each row.


result = True<EOL>for field in [<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:4>, <NUM_LIT:5>, <NUM_LIT:6>]:<EOL><INDENT>if abs(seg1[<NUM_LIT:0>][field] - seg2[<NUM_LIT:0>][field]) > <NUM_LIT>:<EOL><INDENT>result = False<EOL><DEDENT><DEDENT>if len(seg1[<NUM_LIT:1>:]) != len(seg2[<NUM_LIT:1>:]):<EOL><INDENT>result = False<EOL><DEDENT>for syn in seg2[<NUM_LIT:1>:]:<EOL><INDENT>if syn[<NUM_LIT:2>] <= <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>result = False<EOL><DEDENT><DEDENT>if result == True:<EOL><INDENT>for syn in seg1[<NUM_LIT:1>:]:<EOL><INDENT>if syn[<NUM_LIT:2>] <= <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>result = False<EOL><DEDENT>res = sameSynapse(syn, seg2[<NUM_LIT:1>:])<EOL>if res == False:<EOL><INDENT>result = False<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>

Return True if seg1 and seg2 are identical, ignoring order of synapses


def generateSimpleSequences(nCoinc=<NUM_LIT:10>, seqLength=[<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>], nSeq=<NUM_LIT:100>):

coincList = list(range(nCoinc))<EOL>seqList  = []<EOL>for i in range(nSeq):<EOL><INDENT>if max(seqLength) <= nCoinc:<EOL><INDENT>seqList.append(random.sample(coincList, random.choice(seqLength)))<EOL><DEDENT>else:<EOL><INDENT>len = random.choice(seqLength)<EOL>seq = []<EOL>for x in range(len):<EOL><INDENT>seq.append(random.choice(coincList))<EOL><DEDENT>seqList.append(seq)<EOL><DEDENT><DEDENT>return seqList<EOL>

Generate a set of simple sequences. The elements of the sequences will be
integers from 0 to 'nCoinc'-1. The length of each sequence will be
randomly chosen from the 'seqLength' list.

Parameters:
-----------------------------------------------
nCoinc:      the number of elements available to use in the sequences
seqLength:   a list of possible sequence lengths. The length of each
             sequence will be randomly chosen from here.
nSeq:        The number of sequences to generate

retval:      a list of sequences. Each sequence is itself a list
             containing the coincidence indices for that sequence.


def tmDiff(tm1, tm2, verbosity = <NUM_LIT:0>, relaxSegmentTests =True):

<EOL>if sameTMParams(tm1, tm2) == False:<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>result = True<EOL>if (tm1.activeState['<STR_LIT:t>'] != tm2.activeState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.activeState['<STR_LIT:t>'] != tm2.activeState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if (tm1.predictedState['<STR_LIT:t>'] - tm2.predictedState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.predictedState['<STR_LIT:t>'] != tm2.predictedState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if tm1.getNumSegments() != tm2.getNumSegments():<EOL><INDENT>print("<STR_LIT>", tm1.getNumSegments(), tm2.getNumSegments())<EOL>result = False<EOL><DEDENT>if tm1.getNumSynapses() != tm2.getNumSynapses():<EOL><INDENT>print("<STR_LIT>", tm1.getNumSynapses(), tm2.getNumSynapses())<EOL>tm1.printCells()<EOL>tm2.printCells()<EOL>result = False<EOL><DEDENT>for c in range(tm1.numberOfCols):<EOL><INDENT>for i in range(tm2.cellsPerColumn):<EOL><INDENT>if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):<EOL><INDENT>print("<STR_LIT>",c,i, end='<STR_LIT:U+0020>')<EOL>print(tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i))<EOL>result = False<EOL><DEDENT><DEDENT><DEDENT>if result == True and not relaxSegmentTests:<EOL><INDENT>for c in range(tm1.numberOfCols):<EOL><INDENT>for i in range(tm2.cellsPerColumn):<EOL><INDENT>nSegs = tm1.getNumSegmentsInCell(c, i)<EOL>for segIdx in range(nSegs):<EOL><INDENT>tm1seg = tm1.getSegmentOnCell(c, i, segIdx)<EOL>res = False<EOL>for tm2segIdx in range(nSegs):<EOL><INDENT>tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)<EOL>if sameSegment(tm1seg, tm2seg) == True:<EOL><INDENT>res = True<EOL>break<EOL><DEDENT><DEDENT>if res == False:<EOL><INDENT>print("<STR_LIT>",c,i)<EOL>if verbosity >= <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>tm1.printCell(c, i)<EOL>print("<STR_LIT>")<EOL>tm2.printCell(c, i)<EOL><DEDENT>result = False<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if result == True and (verbosity > <NUM_LIT:1>):<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return result<EOL>

Given two TM instances, list the difference between them and returns False
if there is a difference. This function checks the major parameters. If this
passes (and checkLearn is true) it checks the number of segments on
each cell. If this passes, checks each synapse on each segment.
When comparing C++ and Py, the segments are usually in different orders in the
cells. tmDiff ignores segment order when comparing TM's.


<EOL>numVectors = len(vectors)<EOL>if numSamples is None:<EOL><INDENT>numSamples = numVectors-<NUM_LIT:1><EOL>countOn = list(range(numVectors-<NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>countOn = numpy.random.randint(<NUM_LIT:0>, numVectors-<NUM_LIT:1>, numSamples)<EOL><DEDENT>sigmap = <NUM_LIT:0.0><EOL>for i in countOn:<EOL><INDENT>match = checkMatch(vectors[i], vectors[i+<NUM_LIT:1>], sparse=False)<EOL>if match[<NUM_LIT:1>] != <NUM_LIT:0>:<EOL><INDENT>sigmap += float(match[<NUM_LIT:0>])/match[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>return sigmap / numSamples<EOL>

Returns the stability for the population averaged over multiple time steps

Parameters:
-----------------------------------------------
vectors:          the vectors for which the stability is calculated
numSamples        the number of time steps where stability is counted

At each time step, count the fraction of the active elements which are stable
from the previous step
Average all the fraction


totalLen = <NUM_LIT:0><EOL>for seq in seqList:<EOL><INDENT>totalLen += len(seq)<EOL><DEDENT>vectors = numpy.zeros((totalLen, patternMatrix.shape[<NUM_LIT:1>]), dtype='<STR_LIT:bool>')<EOL>vecOffset = <NUM_LIT:0><EOL>for seq in seqList:<EOL><INDENT>seq = numpy.array(seq, dtype='<STR_LIT>')<EOL>for idx,coinc in enumerate(seq):<EOL><INDENT>vectors[vecOffset] = patternMatrix.getRow(int(coinc))<EOL>vecOffset += <NUM_LIT:1><EOL><DEDENT><DEDENT>return vectors<EOL>

Convert a list of sequences of pattern indices, and a pattern lookup table
    into a an array of patterns

Parameters:
-----------------------------------------------
seq:            the sequence, given as indices into the patternMatrix
patternMatrix:  a SparseMatrix contaning the possible patterns used in
                        the sequence.


def tmDiff2(tm1, tm2, verbosity = <NUM_LIT:0>, relaxSegmentTests =True,<EOL>checkLearn = True, checkStates = True):

<EOL>if sameTMParams(tm1, tm2) == False:<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>tm1Label = "<STR_LIT>" % tm1.__class__.__name__<EOL>tm2Label = "<STR_LIT>" % tm2.__class__.__name__<EOL>result = True<EOL>if checkStates:<EOL><INDENT>if (tm1.infActiveState['<STR_LIT:t>'] != tm2.infActiveState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.infActiveState['<STR_LIT:t>'] != tm2.infActiveState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if (tm1.infPredictedState['<STR_LIT:t>'] - tm2.infPredictedState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.infPredictedState['<STR_LIT:t>'] != tm2.infPredictedState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if checkLearn and (tm1.lrnActiveState['<STR_LIT:t>'] - tm2.lrnActiveState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.lrnActiveState['<STR_LIT:t>'] != tm2.lrnActiveState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if checkLearn and (tm1.lrnPredictedState['<STR_LIT:t>'] - tm2.lrnPredictedState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.lrnPredictedState['<STR_LIT:t>'] != tm2.lrnPredictedState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if checkLearn and abs(tm1.getAvgLearnedSeqLength() - tm2.getAvgLearnedSeqLength()) > <NUM_LIT>:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>print(tm1.getAvgLearnedSeqLength(), "<STR_LIT>", tm2.getAvgLearnedSeqLength())<EOL>result = False<EOL><DEDENT><DEDENT>if tm1.getNumSegments() != tm2.getNumSegments():<EOL><INDENT>print("<STR_LIT>", tm1.getNumSegments(), tm2.getNumSegments())<EOL>result = False<EOL><DEDENT>if tm1.getNumSynapses() != tm2.getNumSynapses():<EOL><INDENT>print("<STR_LIT>", tm1.getNumSynapses(), tm2.getNumSynapses())<EOL>if verbosity >= <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>" % tm1Label, end='<STR_LIT:U+0020>')<EOL>tm1.printCells()<EOL>print("<STR_LIT>" % tm2Label, end='<STR_LIT:U+0020>')<EOL>tm2.printCells()<EOL><DEDENT><DEDENT>for c in range(tm1.numberOfCols):<EOL><INDENT>for i in range(tm2.cellsPerColumn):<EOL><INDENT>if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):<EOL><INDENT>print("<STR_LIT>",c,i, end='<STR_LIT:U+0020>')<EOL>print(tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i))<EOL>result = False<EOL><DEDENT><DEDENT><DEDENT>if result == True and not relaxSegmentTests and checkLearn:<EOL><INDENT>for c in range(tm1.numberOfCols):<EOL><INDENT>for i in range(tm2.cellsPerColumn):<EOL><INDENT>nSegs = tm1.getNumSegmentsInCell(c, i)<EOL>for segIdx in range(nSegs):<EOL><INDENT>tm1seg = tm1.getSegmentOnCell(c, i, segIdx)<EOL>res = False<EOL>for tm2segIdx in range(nSegs):<EOL><INDENT>tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)<EOL>if sameSegment(tm1seg, tm2seg) == True:<EOL><INDENT>res = True<EOL>break<EOL><DEDENT><DEDENT>if res == False:<EOL><INDENT>print("<STR_LIT>",c,i)<EOL>result = False<EOL>if verbosity >= <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % tm1Label, end='<STR_LIT:U+0020>')<EOL>tm1.printCell(c, i)<EOL>print("<STR_LIT>" % tm2Label, end='<STR_LIT:U+0020>')<EOL>tm2.printCell(c, i)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if result == True and (verbosity > <NUM_LIT:1>):<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return result<EOL>

Given two TM instances, list the difference between them and returns False
if there is a difference. This function checks the major parameters. If this
passes (and checkLearn is true) it checks the number of segments on each cell.
If this passes, checks each synapse on each segment.
When comparing C++ and Py, the segments are usually in different orders in the
cells. tmDiff ignores segment order when comparing TM's.

If checkLearn is True, will check learn states as well as all the segments

If checkStates is True, will check the various state arrays


def generateSequences(nPatterns=<NUM_LIT:10>, patternLen=<NUM_LIT>, patternActivity=<NUM_LIT:50>,<EOL>hubs=[<NUM_LIT:2>,<NUM_LIT:6>],  seqLength=[<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>],<EOL>nSimpleSequences=<NUM_LIT:50>,  nHubSequences=<NUM_LIT:50>):

<EOL>patterns = generateCoincMatrix(nCoinc=nPatterns, length=patternLen,<EOL>activity=patternActivity)<EOL>seqList =  generateSimpleSequences(nCoinc=nPatterns, seqLength=seqLength,<EOL>nSeq=nSimpleSequences) +generateHubSequences(nCoinc=nPatterns, hubs=hubs, seqLength=seqLength,<EOL>nSeq=nHubSequences)<EOL>return (seqList, patterns)<EOL>

Generate a set of simple and hub sequences. A simple sequence contains
a randomly chosen set of elements from 0 to 'nCoinc-1'. A hub sequence
always contains a hub element in the middle of it.

Parameters:
-----------------------------------------------
nPatterns:        the number of patterns to use in the sequences.
patternLen:       The number of elements in each pattern
patternActivity:  The number of elements that should be active in
                      each pattern
hubs:             which of the elements will be used as hubs.
seqLength:        a list of possible sequence lengths. The length of each
                      sequence will be randomly chosen from here.
nSimpleSequences: The number of simple sequences to generate
nHubSequences:    The number of hub sequences to generate

retval:           (seqList, patterns)
                  seqList: a list of sequences. Each sequence is itself a list
                                containing the input pattern indices for that sequence.
                  patterns: the input patterns used in the seqList.


<EOL>nonzeros = numpy.array(vector).nonzero()[<NUM_LIT:0>]<EOL>if len(nonzeros) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>if len(nonzeros) == <NUM_LIT:1>:<EOL><INDENT>durations[nonzeros[<NUM_LIT:0>]] = <NUM_LIT:1><EOL>return<EOL><DEDENT>prev = nonzeros[<NUM_LIT:0>]<EOL>onTime = <NUM_LIT:1><EOL>onStartIdx = prev<EOL>endIdx = nonzeros[-<NUM_LIT:1>]<EOL>for idx in nonzeros[<NUM_LIT:1>:]:<EOL><INDENT>if idx != prev+<NUM_LIT:1>:<EOL><INDENT>durations[onStartIdx:onStartIdx+onTime] = list(range(<NUM_LIT:1>,onTime+<NUM_LIT:1>))<EOL>onTime       = <NUM_LIT:1><EOL>onStartIdx = idx<EOL><DEDENT>else:<EOL><INDENT>onTime += <NUM_LIT:1><EOL><DEDENT>prev = idx<EOL><DEDENT>durations[onStartIdx:onStartIdx+onTime] = list(range(<NUM_LIT:1>,onTime+<NUM_LIT:1>))<EOL>

Helper function used by averageOnTimePerTimestep. 'durations' is a vector
which must be the same len as vector. For each "on" in vector, it fills in
the corresponding element of duration with the duration of that "on" signal
up until that time

Parameters:
-----------------------------------------------
vector:     vector of output values over time
durations:  vector same length as 'vector', initialized to 0's.
            This is filled in with the durations of each 'on" signal.

Example:
vector:     11100000001100000000011111100000
durations:  12300000001200000000012345600000


def generateCoincMatrix(nCoinc=<NUM_LIT:10>, length=<NUM_LIT>, activity=<NUM_LIT:50>):

coincMatrix0 = SM32(int(nCoinc), int(length))<EOL>theOnes = numpy.array([<NUM_LIT:1.0>] * activity, dtype=numpy.float32)<EOL>for rowIdx in range(nCoinc):<EOL><INDENT>coinc = numpy.array(random.sample(range(length),<EOL>activity), dtype=numpy.uint32)<EOL>coinc.sort()<EOL>coincMatrix0.setRowFromSparse(rowIdx, coinc, theOnes)<EOL><DEDENT>coincMatrix = SM32(int(nCoinc), int(length))<EOL>coincMatrix.initializeWithFixedNNZR(activity)<EOL>return coincMatrix0<EOL>

Generate a coincidence matrix. This is used to generate random inputs to the
temporal learner and to compare the predicted output against.

It generates a matrix of nCoinc rows, each row has length 'length' and has
a total of 'activity' bits on.

Parameters:
-----------------------------------------------
nCoinc:        the number of rows to generate
length:        the length of each row
activity:      the number of ones to put into each row.


<EOL>totalSamples = len(vectors)<EOL>windowSize = numSamples<EOL>numWindows = <NUM_LIT:0><EOL>pctStable = <NUM_LIT:0><EOL>for wStart in range(<NUM_LIT:0>, totalSamples-windowSize+<NUM_LIT:1>):<EOL><INDENT>data = vectors[wStart:wStart+windowSize]<EOL>outputSums = data.sum(axis=<NUM_LIT:0>)<EOL>stableOutputs = (outputSums == windowSize).sum()<EOL>samplePctStable = float(stableOutputs) / data[<NUM_LIT:0>].sum()<EOL>print(samplePctStable)<EOL>pctStable += samplePctStable<EOL>numWindows += <NUM_LIT:1><EOL><DEDENT>return float(pctStable) / numWindows<EOL>

Returns the percent of the outputs that remain completely stable over
N time steps.

Parameters:
-----------------------------------------------
vectors:        the vectors for which the stability is calculated
numSamples:     the number of time steps where stability is counted

For each window of numSamples, count how many outputs are active during
the entire window.


result = True<EOL>for param in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>if getattr(tp1, param) != getattr(tp2,param):<EOL><INDENT>print(param,"<STR_LIT>")<EOL>print(getattr(tp1, param), "<STR_LIT>", getattr(tp2,param))<EOL>result = False<EOL><DEDENT><DEDENT>return result<EOL>

Given two TM instances, see if any parameters are different.


def predictionExtent(inputs, resets, outputs, minOverlapPct=<NUM_LIT>):

<EOL>predCounts = None<EOL>predTotal = <NUM_LIT:0><EOL>nSamples = len(outputs)<EOL>predTotalNotLimited = <NUM_LIT:0><EOL>nSamplesNotLimited = <NUM_LIT:0><EOL>nCols = len(inputs[<NUM_LIT:0>])<EOL>nCellsPerCol = len(outputs[<NUM_LIT:0>]) // nCols<EOL>for idx in range(nSamples):<EOL><INDENT>activeCols = outputs[idx].reshape(nCols, nCellsPerCol).max(axis=<NUM_LIT:1>)<EOL>steps = <NUM_LIT:0><EOL>while (idx+steps+<NUM_LIT:1> < nSamples) and (resets[idx+steps+<NUM_LIT:1>] == <NUM_LIT:0>):<EOL><INDENT>overlap = numpy.logical_and(inputs[idx+steps+<NUM_LIT:1>], activeCols)<EOL>overlapPct = <NUM_LIT> * float(overlap.sum()) / inputs[idx+steps+<NUM_LIT:1>].sum()<EOL>if overlapPct >= minOverlapPct:<EOL><INDENT>steps += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>predCounts = _accumulateFrequencyCounts([steps], predCounts)<EOL>predTotal += steps<EOL>if resets[idx] or((idx+steps+<NUM_LIT:1> < nSamples) and (not resets[idx+steps+<NUM_LIT:1>])):<EOL><INDENT>predTotalNotLimited += steps<EOL>nSamplesNotLimited += <NUM_LIT:1><EOL><DEDENT><DEDENT>return (float(predTotal) / nSamples,<EOL>float(predTotalNotLimited) / nSamplesNotLimited,<EOL>predCounts)<EOL>

Computes the predictive ability of a temporal memory (TM). This routine returns
a value which is the average number of time steps of prediction provided
by the TM. It accepts as input the inputs, outputs, and resets provided to
the TM as well as a 'minOverlapPct' used to evalulate whether or not a
prediction is a good enough match to the actual input.

The 'outputs' are the pooling outputs of the TM. This routine treats each output
as a "manifold" that includes the active columns that should be present in the
next N inputs. It then looks at each successive input and sees if it's active
columns are within the manifold. For each output sample, it computes how
many time steps it can go forward on the input before the input overlap with
the manifold is less then 'minOverlapPct'. It returns the average number of
time steps calculated for each output.

Parameters:
-----------------------------------------------
inputs:          The inputs to the TM. Row 0 contains the inputs from time
                 step 0, row 1 from time step 1, etc.
resets:          The reset input to the TM. Element 0 contains the reset from
                 time step 0, element 1 from time step 1, etc.
outputs:         The pooling outputs from the TM. Row 0 contains the outputs
                 from time step 0, row 1 from time step 1, etc.
minOverlapPct:   How much each input's columns must overlap with the pooling
                 output's columns to be considered a valid prediction.

retval:          (Average number of time steps of prediction over all output
                   samples,
                  Average number of time steps of prediction when we aren't
                   cut short by the end of the sequence,
                  List containing frequency counts of each encountered
                   prediction time)


if(len(SP1._masterConnectedM)!=len(SP2._masterConnectedM)):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>if(len(SP1._masterPotentialM)!=len(SP2._masterPotentialM)):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>if(len(SP1._masterPermanenceM)!=len(SP2._masterPermanenceM)):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>for i in range(<NUM_LIT:0>,len(SP1._masterConnectedM)):<EOL><INDENT>connected1 = SP1._masterConnectedM[i]<EOL>connected2 = SP2._masterConnectedM[i]<EOL>if(connected1!=connected2):<EOL><INDENT>print("<STR_LIT>"  % (i))<EOL>return False<EOL><DEDENT>permanences1 = SP1._masterPermanenceM[i];<EOL>permanences2 = SP2._masterPermanenceM[i];<EOL>if(permanences1!=permanences2):<EOL><INDENT>print("<STR_LIT>" % (i))<EOL>return False<EOL><DEDENT>potential1 = SP1._masterPotentialM[i];<EOL>potential2 = SP2._masterPotentialM[i];<EOL>if(potential1!=potential2):<EOL><INDENT>print("<STR_LIT>" % (i))<EOL>return False<EOL><DEDENT><DEDENT>if(not numpy.array_equal(SP1._firingBoostFactors,SP2._firingBoostFactors)):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>if(not numpy.array_equal(SP1._dutyCycleAfterInh,SP2._dutyCycleAfterInh)):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>if(not numpy.array_equal(SP1._dutyCycleBeforeInh,SP2._dutyCycleBeforeInh)):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>print("<STR_LIT>")<EOL>return True<EOL>

Function that compares two spatial pooler instances. Compares the
static variables between the two poolers to make sure that they are equivalent.

Parameters
-----------------------------------------
SP1 first spatial pooler to be compared

SP2 second spatial pooler to be compared

To establish equality, this function does the following:

1.Compares the connected synapse matrices for each coincidence

2.Compare the potential synapse matrices for each coincidence

3.Compare the permanence matrices for each coincidence

4.Compare the firing boosts between the two poolers.

5.Compare the duty cycles before and after inhibition for both poolers


def generateHubSequences(nCoinc=<NUM_LIT:10>, hubs = [<NUM_LIT:2>,<NUM_LIT:6>], seqLength=[<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>], nSeq=<NUM_LIT:100>):

coincList = list(range(nCoinc))<EOL>for hub in hubs:<EOL><INDENT>coincList.remove(hub)<EOL><DEDENT>seqList = []<EOL>for i in range(nSeq):<EOL><INDENT>length = random.choice(seqLength)-<NUM_LIT:1><EOL>seq = random.sample(coincList,length)<EOL>seq.insert(length//<NUM_LIT:2>, random.choice(hubs))<EOL>seqList.append(seq)<EOL><DEDENT>return seqList<EOL>

Generate a set of hub sequences. These are sequences which contain a hub
element in the middle. The elements of the sequences will be integers
from 0 to 'nCoinc'-1. The hub elements will only appear in the middle of
each sequence. The length of each sequence will be randomly chosen from the
'seqLength' list.

Parameters:
-----------------------------------------------
nCoinc:        the number of elements available to use in the sequences
hubs:          which of the elements will be used as hubs.
seqLength:     a list of possible sequence lengths. The length of each
                      sequence will be randomly chosen from here.
nSeq:          The number of sequences to generate

retval:        a list of sequences. Each sequence is itself a list
              containing the coincidence indices for that sequence.


<EOL>durations = []<EOL>numOnTimes   = <NUM_LIT:0><EOL>totalOnTime = <NUM_LIT:0><EOL>nonzeros = numpy.array(vector).nonzero()[<NUM_LIT:0>]<EOL>if len(nonzeros) == <NUM_LIT:0>:<EOL><INDENT>return (<NUM_LIT:0>, <NUM_LIT:0>, [])<EOL><DEDENT>if len(nonzeros) == <NUM_LIT:1>:<EOL><INDENT>return (<NUM_LIT:1>, <NUM_LIT:1>, [<NUM_LIT:1>])<EOL><DEDENT>prev = nonzeros[<NUM_LIT:0>]<EOL>onTime = <NUM_LIT:1><EOL>endIdx = nonzeros[-<NUM_LIT:1>]<EOL>for idx in nonzeros[<NUM_LIT:1>:]:<EOL><INDENT>if idx != prev+<NUM_LIT:1>:<EOL><INDENT>totalOnTime += onTime<EOL>numOnTimes  += <NUM_LIT:1><EOL>durations.append(onTime)<EOL>onTime       = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>onTime += <NUM_LIT:1><EOL><DEDENT>prev = idx<EOL><DEDENT>totalOnTime += onTime<EOL>numOnTimes  += <NUM_LIT:1><EOL>durations.append(onTime)<EOL>return (totalOnTime, numOnTimes, durations)<EOL>

Returns 3 things for a vector:
  * the total on time
  * the number of runs
  * a list of the durations of each run.

Parameters:
-----------------------------------------------
input stream: 11100000001100000000011111100000
return value: (11, 3, [3, 2, 6])