def _insertOrGetUniqueJobNoRetries(<EOL>self, conn, client, cmdLine, jobHash, clientInfo, clientKey, params,<EOL>minimumWorkers, maximumWorkers, jobType, priority, alreadyRunning):

assert len(client) <= self.CLIENT_MAX_LEN, "<STR_LIT>" + repr(client)<EOL>assert cmdLine, "<STR_LIT>" + repr(cmdLine)<EOL>assert len(jobHash) == self.HASH_MAX_LEN, "<STR_LIT>" % len(jobHash)<EOL>if alreadyRunning:<EOL><INDENT>initStatus = self.STATUS_TESTMODE<EOL><DEDENT>else:<EOL><INDENT>initStatus = self.STATUS_NOTSTARTED<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = (initStatus, client, clientInfo, clientKey, cmdLine, params,<EOL>jobHash, minimumWorkers, maximumWorkers, priority, jobType)<EOL>numRowsInserted = conn.cursor.execute(query, sqlParams)<EOL>jobID = <NUM_LIT:0><EOL>if numRowsInserted == <NUM_LIT:1>:<EOL><INDENT>conn.cursor.execute('<STR_LIT>')<EOL>jobID = conn.cursor.fetchall()[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if jobID == <NUM_LIT:0>:<EOL><INDENT>self._logger.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobType, client, _abbreviate(clientInfo, <NUM_LIT:32>), clientKey, jobHash,<EOL>cmdLine)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert numRowsInserted == <NUM_LIT:0>, repr(numRowsInserted)<EOL><DEDENT>if jobID == <NUM_LIT:0>:<EOL><INDENT>row = self._getOneMatchingRowNoRetries(<EOL>self._jobs, conn, dict(client=client, job_hash=jobHash), ['<STR_LIT>'])<EOL>assert row is not None<EOL>assert len(row) == <NUM_LIT:1>, '<STR_LIT>' + repr(len(row))<EOL>jobID = row[<NUM_LIT:0>]<EOL><DEDENT>if alreadyRunning:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, (self._connectionID, jobID))<EOL><DEDENT>return jobID<EOL>

Attempt to insert a row with the given parameters into the jobs table.
        Return jobID of the inserted row, or of an existing row with matching
        client/jobHash key.

        The combination of client and jobHash are expected to be unique (enforced
        by a unique index on the two columns).

        NOTE: It's possibe that this or another process (on this or another machine)
         already inserted a row with matching client/jobHash key (e.g.,
         StreamMgr). This may also happen undetected by this function due to a
         partially-successful insert operation (e.g., row inserted, but then
         connection was lost while reading response) followed by retries either of
         this function or in SteadyDB module.

        Parameters:
        ----------------------------------------------------------------
        conn:            Owned connection acquired from ConnectionFactory.get()
        client:          Name of the client submitting the job
        cmdLine:         Command line to use to launch each worker process; must be
                          a non-empty string
        jobHash:         unique hash of this job. The caller must insure that this,
                          together with client, uniquely identifies this job request
                          for the purposes of detecting duplicates.
        clientInfo:      JSON encoded dict of client specific information.
        clientKey:       Foreign key.
        params:          JSON encoded dict of the parameters for the job. This
                          can be fetched out of the database by the worker processes
                          based on the jobID.
        minimumWorkers:  minimum number of workers design at a time.
        maximumWorkers:  maximum number of workers desired at a time.
        priority:        Job scheduling priority; 0 is the default priority (
                          ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are
                          higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),
                          and negative values are lower priority (down to
                          ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will
                          be scheduled to run at the expense of the lower-priority
                          jobs, and higher-priority job tasks will preempt those
                          with lower priority if there is inadequate supply of
                          scheduling slots. Excess lower priority job tasks will
                          starve as long as slot demand exceeds supply. Most jobs
                          should be scheduled with DEFAULT_JOB_PRIORITY. System jobs
                          that must run at all cost, such as Multi-Model-Master,
                          should be scheduled with MAX_JOB_PRIORITY.
        alreadyRunning:  Used for unit test purposes only. This inserts the job
                          in the running state. It is used when running a worker
                          in standalone mode without hadoop- it gives it a job
                          record to work with.

        retval:           jobID of the inserted jobs row, or of an existing jobs row
                           with matching client/jobHash key


def _resumeJobNoRetries(self, conn, jobID, alreadyRunning):

<EOL>if alreadyRunning:<EOL><INDENT>initStatus = self.STATUS_TESTMODE<EOL><DEDENT>else:<EOL><INDENT>initStatus = self.STATUS_NOTSTARTED<EOL><DEDENT>assignments = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>]<EOL>assignmentValues = [initStatus]<EOL>if alreadyRunning:<EOL><INDENT>assignments += ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>assignmentValues.append(self._connectionID)<EOL><DEDENT>else:<EOL><INDENT>assignments += ['<STR_LIT>', '<STR_LIT>']<EOL><DEDENT>assignments = '<STR_LIT:U+002CU+0020>'.join(assignments)<EOL>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, assignments)<EOL>sqlParams = assignmentValues + [jobID, self.STATUS_COMPLETED]<EOL>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsAffected <= <NUM_LIT:1>, repr(numRowsAffected)<EOL>if numRowsAffected == <NUM_LIT:0>:<EOL><INDENT>self._logger.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", jobID)<EOL><DEDENT>return<EOL>

Resumes processing of an existing job that is presently in the
        STATUS_COMPLETED state.

        NOTE: this is primarily for resuming suspended Production and Stream Jobs; DO
         NOT use it on Hypersearch jobs.

        This prepares an existing job entry to resume processing. The CJM is always
        periodically sweeping the jobs table and when it finds a job that is ready
        to run, it will proceed to start it up on Hadoop.

        Parameters:
        ----------------------------------------------------------------
        conn:            Owned connection acquired from ConnectionFactory.get()
        jobID:          jobID of the job to resume
        alreadyRunning: Used for unit test purposes only. This inserts the job
                         in the running state. It is used when running a worker
                         in standalone mode without hadoop.

        raises:         Throws a RuntimeError if no rows are affected. This could
                        either be because:
                          1) Because there was not matching jobID
                          2) or if the status of the job was not STATUS_COMPLETED.

        retval:            nothing


return self._connectionID<EOL>

Return our connection ID. This can be used for worker identification
        purposes.

        NOTE: the actual MySQL connection ID used in queries may change from time
         to time if connection is re-acquired (e.g., upon MySQL server restart) or
         when more than one entry from the connection pool has been used (e.g.,
         multi-threaded apps)


<EOL>self.jobCancel(jobID)<EOL>return<EOL>

Requests a job to be suspended

        NOTE: this is primarily for suspending Production Jobs; DO NOT use
        it on Hypersearch jobs. For canceling any job type, use jobCancel() instead!

        Parameters:
        ----------------------------------------------------------------
        jobID:          jobID of the job to resume

        retval:            nothing


@logExceptions(_LOGGER)<EOL><INDENT>def jobResume(self, jobID, alreadyRunning=False):<DEDENT>

row = self.jobGetFields(jobID, ['<STR_LIT:status>'])<EOL>(jobStatus,) = row<EOL>if jobStatus != self.STATUS_COMPLETED:<EOL><INDENT>raise RuntimeError(("<STR_LIT>"<EOL>"<STR_LIT>") % (jobID, jobStatus))<EOL><DEDENT>@g_retrySQL<EOL>def resumeWithRetries():<EOL><INDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>self._resumeJobNoRetries(conn, jobID, alreadyRunning)<EOL><DEDENT><DEDENT>resumeWithRetries()<EOL>return<EOL>

Resumes processing of an existing job that is presently in the
        STATUS_COMPLETED state.

        NOTE: this is primarily for resuming suspended Production Jobs; DO NOT use
        it on Hypersearch jobs.

        NOTE: The job MUST be in the STATUS_COMPLETED state at the time of this
        call, otherwise an exception will be raised.

        This prepares an existing job entry to resume processing. The CJM is always
        periodically sweeping the jobs table and when it finds a job that is ready
        to run, will proceed to start it up on Hadoop.

        Parameters:
        ----------------------------------------------------------------
        job:            jobID of the job to resume
        alreadyRunning: Used for unit test purposes only. This inserts the job
                         in the running state. It is used when running a worker
                         in standalone mode without hadoop.

        raises:         Throws a RuntimeError if no rows are affected. This could
                        either be because:
                          1) Because there was not matching jobID
                          2) or if the status of the job was not STATUS_COMPLETED.

        retval:            nothing


@logExceptions(_LOGGER)<EOL><INDENT>def jobInsert(self, client, cmdLine, clientInfo='<STR_LIT>', clientKey='<STR_LIT>', params='<STR_LIT>',<EOL>alreadyRunning=False, minimumWorkers=<NUM_LIT:0>, maximumWorkers=<NUM_LIT:0>,<EOL>jobType='<STR_LIT>', priority=DEFAULT_JOB_PRIORITY):<DEDENT>

jobHash = self._normalizeHash(uuid.uuid1().bytes)<EOL>@g_retrySQL<EOL>def insertWithRetries():<EOL><INDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>return self._insertOrGetUniqueJobNoRetries(<EOL>conn, client=client, cmdLine=cmdLine, jobHash=jobHash,<EOL>clientInfo=clientInfo, clientKey=clientKey, params=params,<EOL>minimumWorkers=minimumWorkers, maximumWorkers=maximumWorkers,<EOL>jobType=jobType, priority=priority, alreadyRunning=alreadyRunning)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>jobID = insertWithRetries()<EOL><DEDENT>except:<EOL><INDENT>self._logger.exception(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobType, client, _abbreviate(clientInfo, <NUM_LIT>), clientKey, jobHash,<EOL>cmdLine)<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>self._logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, jobType, client, _abbreviate(clientInfo, <NUM_LIT>), clientKey,<EOL>jobHash, cmdLine)<EOL><DEDENT>return jobID<EOL>

Add an entry to the jobs table for a new job request. This is called by
        clients that wish to startup a new job, like a Hypersearch, stream job, or
        specific model evaluation from the engine.

        This puts a new entry into the jobs table. The CJM is always periodically
        sweeping the jobs table and when it finds a new job, will proceed to start it
        up on Hadoop.

        Parameters:
        ----------------------------------------------------------------
        client:          Name of the client submitting the job
        cmdLine:         Command line to use to launch each worker process; must be
                          a non-empty string
        clientInfo:      JSON encoded dict of client specific information.
        clientKey:       Foreign key.
        params:          JSON encoded dict of the parameters for the job. This
                          can be fetched out of the database by the worker processes
                          based on the jobID.
        alreadyRunning:  Used for unit test purposes only. This inserts the job
                          in the running state. It is used when running a worker
                          in standalone mode without hadoop - it gives it a job
                          record to work with.
        minimumWorkers:  minimum number of workers design at a time.
        maximumWorkers:  maximum number of workers desired at a time.
        jobType:         The type of job that this is. This should be one of the
                          JOB_TYPE_XXXX enums. This is needed to allow a standard
                          way of recognizing a job's function and capabilities.
        priority:        Job scheduling priority; 0 is the default priority (
                          ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are
                          higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),
                          and negative values are lower priority (down to
                          ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will
                          be scheduled to run at the expense of the lower-priority
                          jobs, and higher-priority job tasks will preempt those
                          with lower priority if there is inadequate supply of
                          scheduling slots. Excess lower priority job tasks will
                          starve as long as slot demand exceeds supply. Most jobs
                          should be scheduled with DEFAULT_JOB_PRIORITY. System jobs
                          that must run at all cost, such as Multi-Model-Master,
                          should be scheduled with MAX_JOB_PRIORITY.

        retval:          jobID - unique ID assigned to this job


@logExceptions(_LOGGER)<EOL><INDENT>def jobInsertUnique(self, client, cmdLine, jobHash, clientInfo='<STR_LIT>',<EOL>clientKey='<STR_LIT>', params='<STR_LIT>', minimumWorkers=<NUM_LIT:0>,<EOL>maximumWorkers=<NUM_LIT:0>, jobType='<STR_LIT>',<EOL>priority=DEFAULT_JOB_PRIORITY):<DEDENT>

assert cmdLine, "<STR_LIT>" + repr(cmdLine)<EOL>@g_retrySQL<EOL>def insertUniqueWithRetries():<EOL><INDENT>jobHashValue = self._normalizeHash(jobHash)<EOL>jobID = None<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>row = self._getOneMatchingRowNoRetries(<EOL>self._jobs, conn, dict(client=client, job_hash=jobHashValue),<EOL>['<STR_LIT>', '<STR_LIT:status>'])<EOL>if row is not None:<EOL><INDENT>(jobID, status) = row<EOL>if status == self.STATUS_COMPLETED:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = (clientInfo, clientKey, cmdLine, params,<EOL>minimumWorkers, maximumWorkers, priority,<EOL>jobType, jobID, self.STATUS_COMPLETED)<EOL>numRowsUpdated = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsUpdated <= <NUM_LIT:1>, repr(numRowsUpdated)<EOL>if numRowsUpdated == <NUM_LIT:0>:<EOL><INDENT>self._logger.info(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>", jobID)<EOL><DEDENT>self._resumeJobNoRetries(conn, jobID, alreadyRunning=False)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>jobID = self._insertOrGetUniqueJobNoRetries(<EOL>conn, client=client, cmdLine=cmdLine, jobHash=jobHashValue,<EOL>clientInfo=clientInfo, clientKey=clientKey, params=params,<EOL>minimumWorkers=minimumWorkers, maximumWorkers=maximumWorkers,<EOL>jobType=jobType, priority=priority, alreadyRunning=False)<EOL><DEDENT>return jobID<EOL><DEDENT><DEDENT>try:<EOL><INDENT>jobID = insertUniqueWithRetries()<EOL><DEDENT>except:<EOL><INDENT>self._logger.exception(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobType, client, _abbreviate(clientInfo, <NUM_LIT>), clientKey, jobHash,<EOL>cmdLine)<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>self._logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, jobType, client, _abbreviate(clientInfo, <NUM_LIT>), clientKey,<EOL>jobHash, cmdLine)<EOL><DEDENT>return jobID<EOL>

Add an entry to the jobs table for a new job request, but only if the
        same job, by the same client is not already running. If the job is already
        running, or queued up to run, this call does nothing. If the job does not
        exist in the jobs table or has completed, it will be inserted and/or started
        up again.

        This method is called by clients, like StreamMgr, that wish to only start up
        a job if it hasn't already been started up.

        Parameters:
        ----------------------------------------------------------------
        client:          Name of the client submitting the job
        cmdLine:         Command line to use to launch each worker process; must be
                          a non-empty string
        jobHash:         unique hash of this job. The client must insure that this
                          uniquely identifies this job request for the purposes
                          of detecting duplicates.
        clientInfo:      JSON encoded dict of client specific information.
        clientKey:       Foreign key.
        params:          JSON encoded dict of the parameters for the job. This
                          can be fetched out of the database by the worker processes
                          based on the jobID.
        minimumWorkers:  minimum number of workers design at a time.
        maximumWorkers:  maximum number of workers desired at a time.
        jobType:         The type of job that this is. This should be one of the
                          JOB_TYPE_XXXX enums. This is needed to allow a standard
                          way of recognizing a job's function and capabilities.
        priority:        Job scheduling priority; 0 is the default priority (
                          ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are
                          higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),
                          and negative values are lower priority (down to
                          ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will
                          be scheduled to run at the expense of the lower-priority
                          jobs, and higher-priority job tasks will preempt those
                          with lower priority if there is inadequate supply of
                          scheduling slots. Excess lower priority job tasks will
                          starve as long as slot demand exceeds supply. Most jobs
                          should be scheduled with DEFAULT_JOB_PRIORITY. System jobs
                          that must run at all cost, such as Multi-Model-Master,
                          should be scheduled with MAX_JOB_PRIORITY.

        retval:          jobID of the newly inserted or existing job.


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = [self.STATUS_RUNNING, self._connectionID,<EOL>jobID, self.STATUS_NOTSTARTED]<EOL>numRowsUpdated = conn.cursor.execute(query, sqlParams)<EOL>if numRowsUpdated != <NUM_LIT:1>:<EOL><INDENT>self._logger.warn('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', numRowsUpdated)<EOL><DEDENT><DEDENT>return<EOL>

Place the given job in STATUS_RUNNING mode; the job is expected to be
        STATUS_NOTSTARTED.

        NOTE: this function was factored out of jobStartNext because it's also
         needed for testing (e.g., test_client_jobs_dao.py)


<EOL>row = self._getOneMatchingRowWithRetries(<EOL>self._jobs, dict(status=self.STATUS_NOTSTARTED), ['<STR_LIT>'])<EOL>if row is None:<EOL><INDENT>return None<EOL><DEDENT>(jobID,) = row<EOL>self._startJobWithRetries(jobID)<EOL>return jobID<EOL>

For use only by Nupic Scheduler (also known as ClientJobManager) Look
        through the jobs table and see if any new job requests have been
        queued up. If so, pick one and mark it as starting up and create the
        model table to hold the results

        Parameters:
        ----------------------------------------------------------------
        retval:    jobID of the job we are starting up, if found; None if not found


<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, [self._connectionID, self.STATUS_RUNNING])<EOL><DEDENT>return<EOL>

Look through the jobs table and reactivate all that are already in the
        running state by setting their _eng_allocate_new_workers fields to True;
        used by Nupic Scheduler as part of its failure-recovery procedure.


rows = self._getMatchingRowsWithRetries(<EOL>self._jobs, dict(status=self.STATUS_RUNNING),<EOL>[self._jobs.pubToDBNameDict[f]<EOL>for f in self._jobs.jobDemandNamedTuple._fields])<EOL>return [self._jobs.jobDemandNamedTuple._make(r) for r in rows]<EOL>

Look through the jobs table and get the demand - minimum and maximum
        number of workers requested, if new workers are to be allocated, if there
        are any untended dead workers, for all running jobs.

        Parameters:
        ----------------------------------------------------------------
        retval:      list of ClientJobsDAO._jobs.jobDemandNamedTuple nametuples
                      containing the demand - min and max workers,
                      allocate_new_workers, untended_dead_workers, num_failed_workers
                      for each running (STATUS_RUNNING) job. Empty list when there
                      isn't any demand.


<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, [self.STATUS_COMPLETED])<EOL><DEDENT>return<EOL>

Set cancel field of all currently-running jobs to true.


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, [self.STATUS_COMPLETED])<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return rows[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>

Look through the jobs table and count the running jobs whose
        cancel field is true.

        Parameters:
        ----------------------------------------------------------------
        retval:      A count of running jobs with the cancel field set to true.


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>conn.cursor.execute(query, [self.STATUS_COMPLETED])<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return tuple(r[<NUM_LIT:0>] for r in rows)<EOL>

Look through the jobs table and get the list of running jobs whose
        cancel field is true.

        Parameters:
        ----------------------------------------------------------------
        retval:      A (possibly empty) sequence of running job IDs with cancel field
                      set to true


assert sum(intervals) <= len(data)<EOL>start = <NUM_LIT:0><EOL>for interval in intervals:<EOL><INDENT>end = start + interval<EOL>yield data[start:end]<EOL>start = end<EOL><DEDENT>raise StopIteration<EOL>

Generator to allow iterating slices at dynamic intervals

        Parameters:
        ----------------------------------------------------------------
        data:       Any data structure that supports slicing (i.e. list or tuple)
        *intervals: Iterable of intervals.  The sum of intervals should be less
                    than, or equal to the length of data.


results = ClientJobsDAO.partitionAtIntervals(<EOL>result, [len(nt._fields) for nt in namedTuples])<EOL>return [nt._make(result) for nt, result in zip(namedTuples, results)]<EOL>

Return a list of namedtuples from the result of a join query.  A
        single database result is partitioned at intervals corresponding to the
        fields in namedTuples.  The return value is the result of applying
        namedtuple._make() to each of the partitions, for each of the namedTuples.

        Parameters:
        ----------------------------------------------------------------
        result:         Tuple representing a single result from a database query
        *namedTuples:   List of named tuples.


<EOL>combinedResults = None<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT:U+0020>'.join([<EOL>'<STR_LIT>' % (self.jobsTableName, self.modelsTableName),<EOL>'<STR_LIT>' % self.jobsTableName,<EOL>'<STR_LIT>' % self.modelsTableName,<EOL>'<STR_LIT>'])<EOL>conn.cursor.execute(query, (jobID,))<EOL>if conn.cursor.rowcount > <NUM_LIT:0>:<EOL><INDENT>combinedResults = [<EOL>ClientJobsDAO._combineResults(<EOL>result, self._jobs.jobInfoNamedTuple,<EOL>self._models.modelInfoNamedTuple<EOL>) for result in conn.cursor.fetchall()]<EOL><DEDENT><DEDENT>if combinedResults is not None:<EOL><INDENT>return combinedResults<EOL><DEDENT>raise RuntimeError("<STR_LIT>" % (jobID))<EOL>

Get all info about a job, with model details, if available.

        Parameters:
        ----------------------------------------------------------------
        job:    jobID of the job to query
        retval: A sequence of two-tuples if the jobID exists in the jobs
                 table (exeption is raised if it doesn't exist). Each two-tuple
                 contains an instance of jobInfoNamedTuple as the first element and
                 an instance of modelInfoNamedTuple as the second element. NOTE: In
                 the case where there are no matching model rows, a sequence of one
                 two-tuple will still be returned, but the modelInfoNamedTuple
                 fields will be None, and the jobInfoNamedTuple fields will be
                 populated.


row = self._getOneMatchingRowWithRetries(<EOL>self._jobs, dict(job_id=jobID),<EOL>[self._jobs.pubToDBNameDict[n]<EOL>for n in self._jobs.jobInfoNamedTuple._fields])<EOL>if row is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (jobID))<EOL><DEDENT>return self._jobs.jobInfoNamedTuple._make(row)<EOL>

Get all info about a job

        Parameters:
        ----------------------------------------------------------------
        job:    jobID of the job to query
        retval:  namedtuple containing the job info.


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetStatus(self, jobID, status, useConnectionID=True,):<DEDENT>

<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = [status, jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>result = conn.cursor.execute(query, sqlParams)<EOL>if result != <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>jobID, status))<EOL><DEDENT><DEDENT>

Change the status on the given job

        Parameters:
        ----------------------------------------------------------------
        job:        jobID of the job to change status
        status:     new status string (ClientJobsDAO.STATUS_xxxxx)

        useConnectionID: True if the connection id of the calling function
        must be the same as the connection that created the job. Set
        to False for hypersearch workers


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetCompleted(self, jobID, completionReason, completionMsg,<EOL>useConnectionID = True):<DEDENT>

<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.jobsTableName,)<EOL>sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,<EOL>jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>result = conn.cursor.execute(query, sqlParams)<EOL>if result != <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (jobID))<EOL><DEDENT><DEDENT>

Change the status on the given job to completed

        Parameters:
        ----------------------------------------------------------------
        job:                 jobID of the job to mark as completed
        completionReason:    completionReason string
        completionMsg:       completionMsg string

        useConnectionID: True if the connection id of the calling function
        must be the same as the connection that created the job. Set
        to False for hypersearch workers


self._logger.info('<STR_LIT>', jobID)<EOL>self.jobSetFields(jobID, {"<STR_LIT>" : True}, useConnectionID=False)<EOL>

Cancel the given job. This will update the cancel field in the
        jobs table and will result in the job being cancelled.

        Parameters:
        ----------------------------------------------------------------
        jobID:                 jobID of the job to mark as completed

        to False for hypersearch workers


rows = self._getMatchingRowsWithRetries(self._models, dict(job_id=jobID),<EOL>['<STR_LIT>'])<EOL>return [r[<NUM_LIT:0>] for r in rows]<EOL>

Fetch all the modelIDs that correspond to a given jobID; empty sequence
        if none


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>' %  self.jobsTableName<EOL>conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])<EOL>activeJobCount = conn.cursor.fetchone()[<NUM_LIT:0>]<EOL><DEDENT>return activeJobCount<EOL>

Return the number of jobs for the given clientInfo and a status that is
        not completed.


with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>' %  self.jobsTableName<EOL>conn.cursor.execute(query, [clientKey, self.STATUS_COMPLETED])<EOL>activeJobCount = conn.cursor.fetchone()[<NUM_LIT:0>]<EOL><DEDENT>return activeJobCount<EOL>

Return the number of jobs for the given clientKey and a status that is
        not completed.


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def getActiveJobsForClientInfo(self, clientInfo, fields=[]):<DEDENT>

<EOL>dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(['<STR_LIT>'] + dbFields)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>' % (dbFieldsStr, self.jobsTableName)<EOL>conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return rows<EOL>

Fetch jobIDs for jobs in the table with optional fields given a
        specific clientInfo


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def getActiveJobsForClientKey(self, clientKey, fields=[]):<DEDENT>

<EOL>dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(['<STR_LIT>'] + dbFields)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>' % (dbFieldsStr, self.jobsTableName)<EOL>conn.cursor.execute(query, [clientKey, self.STATUS_COMPLETED])<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return rows<EOL>

Fetch jobIDs for jobs in the table with optional fields given a
        specific clientKey


<EOL>dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(['<STR_LIT>'] + dbFields)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>' % (dbFieldsStr, self.jobsTableName)<EOL>conn.cursor.execute(query)<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return rows<EOL>

Fetch jobIDs for jobs in the table with optional fields


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def getFieldsForActiveJobsOfType(self, jobType, fields=[]):<DEDENT>

dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(['<STR_LIT>'] + dbFields)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query ='<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>' % (dbFieldsStr, self.jobsTableName,<EOL>self.modelsTableName)<EOL>conn.cursor.execute(query, [self.STATUS_COMPLETED, jobType])<EOL>return conn.cursor.fetchall()<EOL><DEDENT>

Helper function for querying the models table including relevant job
        info where the job type matches the specified jobType.  Only records for
        which there is a matching jobId in both tables is returned, and only the
        requested fields are returned in each result, assuming that there is not
        a conflict.  This function is useful, for example, in querying a cluster
        for a list of actively running production models (according to the state
        of the client jobs database).  jobType must be one of the JOB_TYPE_XXXX
        enumerations.

        Parameters:
        ----------------------------------------------------------------
        jobType:   jobType enum
        fields:    list of fields to return

        Returns:    List of tuples containing the jobId and requested field values


@logExceptions(_LOGGER)<EOL><INDENT>def jobGetFields(self, jobID, fields):<DEDENT>

<EOL>return self.jobsGetFields([jobID], fields, requireAll=True)[<NUM_LIT:0>][<NUM_LIT:1>]<EOL>

Fetch the values of 1 or more fields from a job record. Here, 'fields'
        is a list with the names of the fields to fetch. The names are the public
        names of the fields (camelBack, not the lower_case_only form as stored in
        the DB).

        Parameters:
        ----------------------------------------------------------------
        jobID:     jobID of the job record
        fields:    list of fields to return

        Returns:    A sequence of field values in the same order as the requested
                     field list -> [field1, field2, ...]


@logExceptions(_LOGGER)<EOL><INDENT>def jobsGetFields(self, jobIDs, fields, requireAll=True):<DEDENT>

assert isinstance(jobIDs, self._SEQUENCE_TYPES)<EOL>assert len(jobIDs) >=<NUM_LIT:1><EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._jobs, dict(job_id=jobIDs),<EOL>['<STR_LIT>'] + [self._jobs.pubToDBNameDict[x] for x in fields])<EOL>if requireAll and len(rows) < len(jobIDs):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (<EOL>(set(jobIDs) - set(r[<NUM_LIT:0>] for r in rows)),))<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Fetch the values of 1 or more fields from a sequence of job records.
        Here, 'fields' is a sequence (list or tuple) with the names of the fields to
        fetch. The names are the public names of the fields (camelBack, not the
        lower_case_only form as stored in the DB).

        WARNING!!!: The order of the results are NOT necessarily in the same order as
        the order of the job IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        jobIDs:        A sequence of jobIDs
        fields:        A list  of fields to return for each jobID

        Returns:      A list of tuples->(jobID, [field1, field2,...])


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetFields(self, jobID, fields, useConnectionID=True,<EOL>ignoreUnchanged=False):<DEDENT>

<EOL>assignmentExpressions = '<STR_LIT:U+002C>'.join(<EOL>["<STR_LIT>" % (self._jobs.pubToDBNameDict[f],) for f in fields.keys()])<EOL>assignmentValues = list(fields.values())<EOL>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, assignmentExpressions,)<EOL>sqlParams = assignmentValues + [jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>result = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if result != <NUM_LIT:1> and not ignoreUnchanged:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>""<STR_LIT>" % (<EOL>assignmentExpressions, jobID, self._connectionID, result, query))<EOL><DEDENT>

Change the values of 1 or more fields in a job. Here, 'fields' is a
        dict with the name/value pairs to change. The names are the public names of
        the fields (camelBack, not the lower_case_only form as stored in the DB).
        This method is for private use by the ClientJobManager only.

        Parameters:
        ----------------------------------------------------------------
        jobID:     jobID of the job record

        fields:    dictionary of fields to change

        useConnectionID: True if the connection id of the calling function
        must be the same as the connection that created the job. Set
        to False for hypersearch workers

        ignoreUnchanged: The default behavior is to throw a
        RuntimeError if no rows are affected. This could either be
        because:
          1) Because there was not matching jobID
          2) or if the data to update matched the data in the DB exactly.

        Set this parameter to True if you expect case 2 and wish to
        supress the error.


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobSetFieldIfEqual(self, jobID, fieldName, newValue, curValue):<DEDENT>

<EOL>dbFieldName = self._jobs.pubToDBNameDict[fieldName]<EOL>conditionValue = []<EOL>if isinstance(curValue, bool):<EOL><INDENT>conditionExpression = '<STR_LIT>' % (<EOL>dbFieldName, {True:'<STR_LIT>', False:'<STR_LIT>'}[curValue])<EOL><DEDENT>elif curValue is None:<EOL><INDENT>conditionExpression = '<STR_LIT>' % (dbFieldName,)<EOL><DEDENT>else:<EOL><INDENT>conditionExpression = '<STR_LIT>' % (dbFieldName,)<EOL>conditionValue.append(curValue)<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, dbFieldName, conditionExpression)<EOL>sqlParams = [newValue, jobID] + conditionValue<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>result = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>return (result == <NUM_LIT:1>)<EOL>

Change the value of 1 field in a job to 'newValue', but only if the
        current value matches 'curValue'. The 'fieldName' is the public name of
        the field (camelBack, not the lower_case_only form as stored in the DB).

        This method is used for example by HypersearcWorkers to update the
        engWorkerState field periodically. By qualifying on curValue, it insures
        that only 1 worker at a time is elected to perform the next scheduled
        periodic sweep of the models.

        Parameters:
        ----------------------------------------------------------------
        jobID:        jobID of the job record to modify
        fieldName:    public field name of the field
        newValue:     new value of the field to set
        curValue:     current value to qualify against

        retval:       True if we successfully modified the field
                      False if curValue did not match


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobIncrementIntField(self, jobID, fieldName, increment=<NUM_LIT:1>,<EOL>useConnectionID=False):<DEDENT>

<EOL>dbFieldName = self._jobs.pubToDBNameDict[fieldName]<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>'% (self.jobsTableName, dbFieldName, dbFieldName)<EOL>sqlParams = [increment, jobID]<EOL>if useConnectionID:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>result = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if result != <NUM_LIT:1>:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>""<STR_LIT>" % (<EOL>dbFieldName, jobID, self._connectionID, result, query))<EOL><DEDENT>

Incremet the value of 1 field in a job by increment. The 'fieldName' is
        the public name of the field (camelBack, not the lower_case_only form as
        stored in the DB).

        This method is used for example by HypersearcWorkers to update the
        engWorkerState field periodically. By qualifying on curValue, it insures
        that only 1 worker at a time is elected to perform the next scheduled
        periodic sweep of the models.

        Parameters:
        ----------------------------------------------------------------
        jobID:        jobID of the job record to modify
        fieldName:    public field name of the field
        increment:    increment is added to the current value of the field


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def jobUpdateResults(self, jobID, results):<DEDENT>

with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>' % (self.jobsTableName,)<EOL>conn.cursor.execute(query, [results, jobID])<EOL><DEDENT>

Update the results string and last-update-time fields of a model.

        Parameters:
        ----------------------------------------------------------------
        jobID:      job ID of model to modify
        results:    new results (json dict string)


self._logger.info('<STR_LIT>',<EOL>self.modelsTableName)<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>' % (self.modelsTableName)<EOL>conn.cursor.execute(query)<EOL><DEDENT>


@logExceptions(_LOGGER)<EOL><INDENT>def modelInsertAndStart(self, jobID, params, paramsHash, particleHash=None):<DEDENT>

<EOL>if particleHash is None:<EOL><INDENT>particleHash = paramsHash<EOL><DEDENT>paramsHash = self._normalizeHash(paramsHash)<EOL>particleHash = self._normalizeHash(particleHash)<EOL>def findExactMatchNoRetries(conn):<EOL><INDENT>return self._getOneMatchingRowNoRetries(<EOL>self._models, conn,<EOL>{'<STR_LIT>':jobID, '<STR_LIT>':paramsHash,<EOL>'<STR_LIT>':particleHash},<EOL>['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>@g_retrySQL<EOL>def findExactMatchWithRetries():<EOL><INDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>return findExactMatchNoRetries(conn)<EOL><DEDENT><DEDENT>row = findExactMatchWithRetries()<EOL>if row is not None:<EOL><INDENT>return (row[<NUM_LIT:0>], False)<EOL><DEDENT>@g_retrySQL<EOL>def insertModelWithRetries():<EOL><INDENT>"""<STR_LIT>"""<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = (jobID, params, self.STATUS_RUNNING, paramsHash,<EOL>particleHash, self._connectionID)<EOL>try:<EOL><INDENT>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>except Exception as e:<EOL><INDENT>if "<STR_LIT>" not in str(e):<EOL><INDENT>raise<EOL><DEDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, paramsHash.encode('<STR_LIT>'),<EOL>particleHash.encode('<STR_LIT>'), e)<EOL><DEDENT>else:<EOL><INDENT>if numRowsAffected == <NUM_LIT:1>:<EOL><INDENT>conn.cursor.execute('<STR_LIT>')<EOL>modelID = conn.cursor.fetchall()[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if modelID != <NUM_LIT:0>:<EOL><INDENT>return (modelID, True)<EOL><DEDENT>else:<EOL><INDENT>self._logger.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>jobID, paramsHash, particleHash)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._logger.error(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>numRowsAffected, jobID, paramsHash, particleHash)<EOL><DEDENT><DEDENT>row = findExactMatchNoRetries(conn)<EOL>if row is not None:<EOL><INDENT>(modelID, connectionID) = row<EOL>return (modelID, connectionID == self._connectionID)<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = [jobID, paramsHash, particleHash]<EOL>numRowsFound = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsFound == <NUM_LIT:1>, (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>') % (jobID, paramsHash, particleHash, numRowsFound)<EOL>(modelID,) = conn.cursor.fetchall()[<NUM_LIT:0>]<EOL>return (modelID, False)<EOL><DEDENT><DEDENT>return insertModelWithRetries()<EOL>

Insert a new unique model (based on params) into the model table in the
        "running" state. This will return two things: whether or not the model was
        actually inserted (i.e. that set of params isn't already in the table) and
        the modelID chosen for that set of params. Even if the model was not
        inserted by this call (it was already there) the modelID of the one already
        inserted is returned.

        Parameters:
        ----------------------------------------------------------------
        jobID:            jobID of the job to add models for
        params:           params for this model
        paramsHash        hash of the params, generated by the worker
        particleHash      hash of the particle info (for PSO). If not provided,
                          then paramsHash will be used.

        retval:           (modelID, wasInserted)
                          modelID: the model ID for this set of params
                          wasInserted: True if this call ended up inserting the
                          new model. False if this set of params was already in
                          the model table.


assert isinstance(modelIDs, self._SEQUENCE_TYPES), (<EOL>"<STR_LIT>") % (type(modelIDs),)<EOL>assert modelIDs, "<STR_LIT>"<EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._models, dict(model_id=modelIDs),<EOL>[self._models.pubToDBNameDict[f]<EOL>for f in self._models.modelInfoNamedTuple._fields])<EOL>results = [self._models.modelInfoNamedTuple._make(r) for r in rows]<EOL>assert len(results) == len(modelIDs), "<STR_LIT>" % (<EOL>set(modelIDs) - set(r.modelId for r in results))<EOL>return results<EOL>

Get ALL info for a set of models

        WARNING!!!: The order of the results are NOT necessarily in the same order as
        the order of the model IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        modelIDs:    list of model IDs
        retval:      list of nametuples containing all the fields stored for each
                        model.


@logExceptions(_LOGGER)<EOL><INDENT>def modelsGetFields(self, modelIDs, fields):<DEDENT>

assert len(fields) >= <NUM_LIT:1>, '<STR_LIT>'<EOL>isSequence = isinstance(modelIDs, self._SEQUENCE_TYPES)<EOL>if isSequence:<EOL><INDENT>assert len(modelIDs) >=<NUM_LIT:1>, '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>modelIDs = [modelIDs]<EOL><DEDENT>rows = self._getMatchingRowsWithRetries(<EOL>self._models, dict(model_id=modelIDs),<EOL>['<STR_LIT>'] + [self._models.pubToDBNameDict[f] for f in fields])<EOL>if len(rows) < len(modelIDs):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (<EOL>(set(modelIDs) - set(r[<NUM_LIT:0>] for r in rows)),))<EOL><DEDENT>if not isSequence:<EOL><INDENT>return list(rows[<NUM_LIT:0>][<NUM_LIT:1>:])<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Fetch the values of 1 or more fields from a sequence of model records.
        Here, 'fields' is a list with the names of the fields to fetch. The names
        are the public names of the fields (camelBack, not the lower_case_only form
        as stored in the DB).

        WARNING!!!: The order of the results are NOT necessarily in the same order
        as the order of the model IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        modelIDs:      A single modelID or sequence of modelIDs
        fields:        A list  of fields to return

        Returns:  If modelIDs is a sequence:
                    a list of tuples->(modelID, [field1, field2,...])
                  If modelIDs is a single modelID:
                    a list of field values->[field1, field2,...]


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelsGetFieldsForJob(self, jobID, fields, ignoreKilled=False):<DEDENT>

assert len(fields) >= <NUM_LIT:1>, '<STR_LIT>'<EOL>dbFields = [self._models.pubToDBNameDict[x] for x in fields]<EOL>dbFieldsStr = '<STR_LIT:U+002C>'.join(dbFields)<EOL>query = '<STR_LIT>''<STR_LIT>'% (dbFieldsStr, self.modelsTableName)<EOL>sqlParams = [jobID]<EOL>if ignoreKilled:<EOL><INDENT>query += '<STR_LIT>'<EOL>sqlParams.append(self.CMPL_REASON_KILLED)<EOL><DEDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>conn.cursor.execute(query, sqlParams)<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>if rows is None:<EOL><INDENT>self._logger.error("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>query, traceback.format_exc())<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Gets the specified fields for all the models for a single job. This is
        similar to modelsGetFields

        Parameters:
        ----------------------------------------------------------------
        jobID:              jobID for the models to be searched
        fields:             A list  of fields to return
        ignoreKilled:       (True/False). If True, this will ignore models that
                            have been killed

        Returns: a (possibly empty) list of tuples as follows
          [
            (model_id1, [field1, ..., fieldn]),
            (model_id2, [field1, ..., fieldn]),
            (model_id3, [field1, ..., fieldn])
                        ...
          ]

        NOTE: since there is a window of time between a job getting inserted into
         jobs table and the job's worker(s) starting up and creating models, an
         empty-list result is one of the normal outcomes.


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelsGetFieldsForCheckpointed(self, jobID, fields):<DEDENT>

assert len(fields) >= <NUM_LIT:1>, "<STR_LIT>"<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>dbFields = [self._models.pubToDBNameDict[f] for f in fields]<EOL>dbFieldStr = "<STR_LIT:U+002CU+0020>".join(dbFields)<EOL>query = '<STR_LIT>''<STR_LIT>'.format(<EOL>fields=dbFieldStr, models=self.modelsTableName)<EOL>conn.cursor.execute(query, [jobID])<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>return [(r[<NUM_LIT:0>], list(r[<NUM_LIT:1>:])) for r in rows]<EOL>

Gets fields from all models in a job that have been checkpointed. This is
used to figure out whether or not a new model should be checkpointed.

Parameters:
-----------------------------------------------------------------------
jobID:                    The jobID for the models to be searched
fields:                   A list of fields to return

Returns: a (possibly-empty) list of tuples as follows
  [
    (model_id1, [field1, ..., fieldn]),
    (model_id2, [field1, ..., fieldn]),
    (model_id3, [field1, ..., fieldn])
                ...
  ]


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelSetFields(self, modelID, fields, ignoreUnchanged = False):<DEDENT>

<EOL>assignmentExpressions = '<STR_LIT:U+002C>'.join(<EOL>'<STR_LIT>' % (self._models.pubToDBNameDict[f],) for f in fields.keys())<EOL>assignmentValues = list(fields.values())<EOL>query = '<STR_LIT>''<STR_LIT>'% (self.modelsTableName, assignmentExpressions)<EOL>sqlParams = assignmentValues + [modelID]<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>numAffectedRows = conn.cursor.execute(query, sqlParams)<EOL>self._logger.debug("<STR_LIT>",<EOL>numAffectedRows, query, sqlParams)<EOL><DEDENT>if numAffectedRows != <NUM_LIT:1> and not ignoreUnchanged:<EOL><INDENT>raise RuntimeError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>") % (<EOL>fields, modelID, self._connectionID, numAffectedRows, query,<EOL>sqlParams,))<EOL><DEDENT>

Change the values of 1 or more fields in a model. Here, 'fields' is a
        dict with the name/value pairs to change. The names are the public names of
        the fields (camelBack, not the lower_case_only form as stored in the DB).

        Parameters:
        ----------------------------------------------------------------
        jobID:     jobID of the job record

        fields:    dictionary of fields to change

        ignoreUnchanged: The default behavior is to throw a
        RuntimeError if no rows are affected. This could either be
        because:
          1) Because there was no matching modelID
          2) or if the data to update matched the data in the DB exactly.

        Set this parameter to True if you expect case 2 and wish to
        supress the error.


assert isinstance(modelIDs, self._SEQUENCE_TYPES), (<EOL>"<STR_LIT>") % (type(modelIDs),)<EOL>assert len(modelIDs) >= <NUM_LIT:1>, "<STR_LIT>"<EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._models, {'<STR_LIT>' : modelIDs},<EOL>[self._models.pubToDBNameDict[f]<EOL>for f in self._models.getParamsNamedTuple._fields])<EOL>assert len(rows) == len(modelIDs), "<STR_LIT>" % (<EOL>(set(modelIDs) - set(r[<NUM_LIT:0>] for r in rows)),)<EOL>return [self._models.getParamsNamedTuple._make(r) for r in rows]<EOL>

Get the params and paramsHash for a set of models.

        WARNING!!!: The order of the results are NOT necessarily in the same order as
        the order of the model IDs passed in!!!

        Parameters:
        ----------------------------------------------------------------
        modelIDs:    list of model IDs
        retval:      list of result namedtuples defined in
                      ClientJobsDAO._models.getParamsNamedTuple. Each tuple
                      contains: (modelId, params, engParamsHash)


assert isinstance(modelIDs, self._SEQUENCE_TYPES), (<EOL>"<STR_LIT>") % type(modelIDs)<EOL>assert len(modelIDs) >= <NUM_LIT:1>, "<STR_LIT>"<EOL>rows = self._getMatchingRowsWithRetries(<EOL>self._models, {'<STR_LIT>' : modelIDs},<EOL>[self._models.pubToDBNameDict[f]<EOL>for f in self._models.getResultAndStatusNamedTuple._fields])<EOL>assert len(rows) == len(modelIDs), "<STR_LIT>" % (<EOL>(set(modelIDs) - set(r[<NUM_LIT:0>] for r in rows)),)<EOL>return [self._models.getResultAndStatusNamedTuple._make(r) for r in rows]<EOL>

Get the results string and other status fields for a set of models.

        WARNING!!!: The order of the results are NOT necessarily in the same order
        as the order of the model IDs passed in!!!

        For each model, this returns a tuple containing:
         (modelID, results, status, updateCounter, numRecords, completionReason,
             completionMsg, engParamsHash

        Parameters:
        ----------------------------------------------------------------
        modelIDs:    list of model IDs
        retval:      list of result tuples. Each tuple contains:
                        (modelID, results, status, updateCounter, numRecords,
                          completionReason, completionMsg, engParamsHash)


rows = self._getMatchingRowsWithRetries(<EOL>self._models, {'<STR_LIT>' : jobID},<EOL>[self._models.pubToDBNameDict[f]<EOL>for f in self._models.getUpdateCountersNamedTuple._fields])<EOL>return [self._models.getUpdateCountersNamedTuple._make(r) for r in rows]<EOL>

Return info on all of the models that are in already in the models
        table for a given job. For each model, this returns a tuple
        containing: (modelID, updateCounter).

        Note that we don't return the results for all models, since the results
        string could be quite large. The information we are returning is
        just 2 integer fields.

        Parameters:
        ----------------------------------------------------------------
        jobID:      jobID to query
        retval:     (possibly empty) list of tuples. Each tuple contains:
                      (modelID, updateCounter)


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelUpdateResults(self, modelID, results=None, metricValue =None,<EOL>numRecords=None):<DEDENT>

assignmentExpressions = ['<STR_LIT>',<EOL>'<STR_LIT>']<EOL>assignmentValues = []<EOL>if results is not None:<EOL><INDENT>assignmentExpressions.append('<STR_LIT>')<EOL>assignmentValues.append(results)<EOL><DEDENT>if numRecords is not None:<EOL><INDENT>assignmentExpressions.append('<STR_LIT>')<EOL>assignmentValues.append(numRecords)<EOL><DEDENT>if metricValue is not None and (metricValue==metricValue):<EOL><INDENT>assignmentExpressions.append('<STR_LIT>')<EOL>assignmentValues.append(float(metricValue))<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>'% (self.modelsTableName, '<STR_LIT:U+002C>'.join(assignmentExpressions))<EOL>sqlParams = assignmentValues + [modelID, self._connectionID]<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if numRowsAffected != <NUM_LIT:1>:<EOL><INDENT>raise InvalidConnectionException(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % (modelID,self._connectionID, numRowsAffected,))<EOL><DEDENT>

Update the results string, and/or num_records fields of
        a model. This will fail if the model does not currently belong to this
        client (connection_id doesn't match).

        Parameters:
        ----------------------------------------------------------------
        modelID:      model ID of model to modify
        results:      new results, or None to ignore
        metricValue:  the value of the metric being optimized, or None to ignore
        numRecords:   new numRecords, or None to ignore


@logExceptions(_LOGGER)<EOL><INDENT>@g_retrySQL<EOL>def modelSetCompleted(self, modelID, completionReason, completionMsg,<EOL>cpuTime=<NUM_LIT:0>, useConnectionID=True):<DEDENT>

if completionMsg is None:<EOL><INDENT>completionMsg = '<STR_LIT>'<EOL><DEDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,<EOL>cpuTime, modelID]<EOL>if useConnectionID:<EOL><INDENT>query += "<STR_LIT>"<EOL>sqlParams.append(self._connectionID)<EOL><DEDENT>with ConnectionFactory.get() as conn:<EOL><INDENT>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL><DEDENT>if numRowsAffected != <NUM_LIT:1>:<EOL><INDENT>raise InvalidConnectionException(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % (modelID, self._connectionID, numRowsAffected))<EOL><DEDENT>

Mark a model as completed, with the given completionReason and
        completionMsg. This will fail if the model does not currently belong to this
        client (connection_id doesn't match).

        Parameters:
        ----------------------------------------------------------------
        modelID:             model ID of model to modify
        completionReason:    completionReason string
        completionMsg:       completionMsg string
        cpuTime:             amount of CPU time spent on this model
        useConnectionID:     True if the connection id of the calling function
                              must be the same as the connection that created the
                              job. Set to True for hypersearch workers, which use
                              this mechanism for orphaned model detection.


@logExceptions(_LOGGER)<EOL><INDENT>def modelAdoptNextOrphan(self, jobId, maxUpdateInterval):<DEDENT>

@g_retrySQL<EOL>def findCandidateModelWithRetries():<EOL><INDENT>modelID = None<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = [self.STATUS_RUNNING, jobId, maxUpdateInterval]<EOL>numRows = conn.cursor.execute(query, sqlParams)<EOL>rows = conn.cursor.fetchall()<EOL><DEDENT>assert numRows <= <NUM_LIT:1>, "<STR_LIT>" % numRows<EOL>if numRows == <NUM_LIT:1>:<EOL><INDENT>(modelID,) = rows[<NUM_LIT:0>]<EOL><DEDENT>return modelID<EOL><DEDENT>@g_retrySQL<EOL>def adoptModelWithRetries(modelID):<EOL><INDENT>adopted = False<EOL>with ConnectionFactory.get() as conn:<EOL><INDENT>query = '<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>''<STR_LIT>'% (self.modelsTableName,)<EOL>sqlParams = [self._connectionID, modelID, self.STATUS_RUNNING,<EOL>maxUpdateInterval]<EOL>numRowsAffected = conn.cursor.execute(query, sqlParams)<EOL>assert numRowsAffected <= <NUM_LIT:1>, '<STR_LIT>' % (<EOL>numRowsAffected,)<EOL>if numRowsAffected == <NUM_LIT:1>:<EOL><INDENT>adopted = True<EOL><DEDENT>else:<EOL><INDENT>(status, connectionID) = self._getOneMatchingRowNoRetries(<EOL>self._models, conn, {'<STR_LIT>':modelID},<EOL>['<STR_LIT:status>', '<STR_LIT>'])<EOL>adopted = (status == self.STATUS_RUNNING and<EOL>connectionID == self._connectionID)<EOL><DEDENT><DEDENT>return adopted<EOL><DEDENT>adoptedModelID = None<EOL>while True:<EOL><INDENT>modelID = findCandidateModelWithRetries()<EOL>if modelID is None:<EOL><INDENT>break<EOL><DEDENT>if adoptModelWithRetries(modelID):<EOL><INDENT>adoptedModelID = modelID<EOL>break<EOL><DEDENT><DEDENT>return adoptedModelID<EOL>

Look through the models table for an orphaned model, which is a model
        that is not completed yet, whose _eng_last_update_time is more than
        maxUpdateInterval seconds ago.

        If one is found, change its _eng_worker_conn_id to the current worker's
        and return the model id.

        Parameters:
        ----------------------------------------------------------------
        retval:    modelId of the model we adopted, or None if none found


if verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>", len(anomalyScores))<EOL>print("<STR_LIT>", skipRecords)<EOL>print("<STR_LIT>", anomalyScores[<NUM_LIT:0>:min(<NUM_LIT:20>, len(anomalyScores))])<EOL><DEDENT>if len(anomalyScores) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>aggRecordList, historicalValues, total =  _anomalyScoreMovingAverage(<EOL>anomalyScores,<EOL>windowSize = averagingWindow,<EOL>verbosity = verbosity)<EOL>s = [r[<NUM_LIT:2>] for r in aggRecordList]<EOL>dataValues = numpy.array(s)<EOL>if len(aggRecordList) <= skipRecords:<EOL><INDENT>distributionParams = nullDistribution(verbosity = verbosity)<EOL><DEDENT>else:<EOL><INDENT>distributionParams = estimateNormal(dataValues[skipRecords:])<EOL>s = [r[<NUM_LIT:1>] for r in aggRecordList]<EOL>if all([isinstance(r[<NUM_LIT:1>], numbers.Number) for r in aggRecordList]):<EOL><INDENT>metricValues = numpy.array(s)<EOL>metricDistribution = estimateNormal(metricValues[skipRecords:],<EOL>performLowerBoundCheck=False)<EOL>if metricDistribution["<STR_LIT>"] < <NUM_LIT>:<EOL><INDENT>distributionParams = nullDistribution(verbosity = verbosity)<EOL><DEDENT><DEDENT><DEDENT>likelihoods = numpy.array(dataValues, dtype=float)<EOL>for i, s in enumerate(dataValues):<EOL><INDENT>likelihoods[i] = tailProbability(s, distributionParams)<EOL><DEDENT>filteredLikelihoods = numpy.array(<EOL>_filterLikelihoods(likelihoods) )<EOL>params = {<EOL>"<STR_LIT>":       distributionParams,<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": historicalValues,<EOL>"<STR_LIT>":            total,<EOL>"<STR_LIT>":       averagingWindow,<EOL>},<EOL>"<STR_LIT>":<EOL>list(likelihoods[-min(averagingWindow, len(likelihoods)):]),<EOL>}<EOL>if verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>print(params)<EOL>print("<STR_LIT>", len(likelihoods))<EOL>print("<STR_LIT>", (<EOL>filteredLikelihoods[<NUM_LIT:0>:min(<NUM_LIT:20>, len(filteredLikelihoods))] ))<EOL>print("<STR_LIT>")<EOL><DEDENT>return (filteredLikelihoods, aggRecordList, params)<EOL>

Given a series of anomaly scores, compute the likelihood for each score. This
function should be called once on a bunch of historical anomaly scores for an
initial estimate of the distribution. It should be called again every so often
(say every 50 records) to update the estimate.

:param anomalyScores: a list of records. Each record is a list with the
                      following three elements: [timestamp, value, score]

                      Example::

                          [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]

                      For best results, the list should be between 1000
                      and 10,000 records
:param averagingWindow: integer number of records to average over
:param skipRecords: integer specifying number of records to skip when
                    estimating distributions. If skip records are >=
                    len(anomalyScores), a very broad distribution is returned
                    that makes everything pretty likely.
:param verbosity: integer controlling extent of printouts for debugging

                    0 = none
                    1 = occasional information
                    2 = print every record

:returns: 3-tuple consisting of:

          - likelihoods

            numpy array of likelihoods, one for each aggregated point

          - avgRecordList

            list of averaged input records

          - params

            a small JSON dict that contains the state of the estimator


if verbosity > <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>")<EOL>print("<STR_LIT>", len(anomalyScores))<EOL>print("<STR_LIT>", anomalyScores[<NUM_LIT:0>:min(<NUM_LIT:20>, len(anomalyScores))])<EOL>print("<STR_LIT>", params)<EOL><DEDENT>if len(anomalyScores) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not isValidEstimatorParams(params):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if "<STR_LIT>" not in params:<EOL><INDENT>params["<STR_LIT>"] = [<NUM_LIT:1.0>]<EOL><DEDENT>historicalValues  = params["<STR_LIT>"]["<STR_LIT>"]<EOL>total             = params["<STR_LIT>"]["<STR_LIT>"]<EOL>windowSize        = params["<STR_LIT>"]["<STR_LIT>"]<EOL>aggRecordList = numpy.zeros(len(anomalyScores), dtype=float)<EOL>likelihoods = numpy.zeros(len(anomalyScores), dtype=float)<EOL>for i, v in enumerate(anomalyScores):<EOL><INDENT>newAverage, historicalValues, total = (<EOL>MovingAverage.compute(historicalValues, total, v[<NUM_LIT:2>], windowSize)<EOL>)<EOL>aggRecordList[i] = newAverage<EOL>likelihoods[i]   = tailProbability(newAverage, params["<STR_LIT>"])<EOL><DEDENT>likelihoods2 = params["<STR_LIT>"] + list(likelihoods)<EOL>filteredLikelihoods = _filterLikelihoods(likelihoods2)<EOL>likelihoods[:] = filteredLikelihoods[-len(likelihoods):]<EOL>historicalLikelihoods = likelihoods2[-min(windowSize, len(likelihoods2)):]<EOL>newParams = {<EOL>"<STR_LIT>": params["<STR_LIT>"],<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": historicalValues,<EOL>"<STR_LIT>": total,<EOL>"<STR_LIT>": windowSize,<EOL>},<EOL>"<STR_LIT>": historicalLikelihoods,<EOL>}<EOL>assert len(newParams["<STR_LIT>"]) <= windowSize<EOL>if verbosity > <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>", len(likelihoods))<EOL>print("<STR_LIT>", likelihoods[<NUM_LIT:0>:min(<NUM_LIT:20>, len(likelihoods))])<EOL>print("<STR_LIT>")<EOL><DEDENT>return (likelihoods, aggRecordList, newParams)<EOL>

Compute updated probabilities for anomalyScores using the given params.

:param anomalyScores: a list of records. Each record is a list with the
                      following three elements: [timestamp, value, score]

                      Example::

                          [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]

:param params: the JSON dict returned by estimateAnomalyLikelihoods
:param verbosity: integer controlling extent of printouts for debugging
:type verbosity: int

:returns: 3-tuple consisting of:

          - likelihoods

            numpy array of likelihoods, one for each aggregated point

          - avgRecordList

            list of averaged input records

          - params

            an updated JSON object containing the state of this metric.


redThreshold    = <NUM_LIT:1.0> - redThreshold<EOL>yellowThreshold = <NUM_LIT:1.0> - yellowThreshold<EOL>filteredLikelihoods = [likelihoods[<NUM_LIT:0>]]<EOL>for i, v in enumerate(likelihoods[<NUM_LIT:1>:]):<EOL><INDENT>if v <= redThreshold:<EOL><INDENT>if likelihoods[i] > redThreshold:<EOL><INDENT>filteredLikelihoods.append(v)<EOL><DEDENT>else:<EOL><INDENT>filteredLikelihoods.append(yellowThreshold)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>filteredLikelihoods.append(v)<EOL><DEDENT><DEDENT>return filteredLikelihoods<EOL>

Filter the list of raw (pre-filtered) likelihoods so that we only preserve
sharp increases in likelihood. 'likelihoods' can be a numpy array of floats or
a list of floats.

:returns: A new list of floats likelihoods containing the filtered values.


historicalValues = []<EOL>total = <NUM_LIT:0.0><EOL>averagedRecordList = []    <EOL>for record in anomalyScores:<EOL><INDENT>if not isinstance(record, (list, tuple)) or len(record) != <NUM_LIT:3>:<EOL><INDENT>if verbosity >= <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>", record)<EOL><DEDENT>continue<EOL><DEDENT>avg, historicalValues, total = (<EOL>MovingAverage.compute(historicalValues, total, record[<NUM_LIT:2>], windowSize)<EOL>)<EOL>averagedRecordList.append( [record[<NUM_LIT:0>], record[<NUM_LIT:1>], avg] )<EOL>if verbosity > <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>", record)<EOL>print("<STR_LIT>", [record[<NUM_LIT:0>], record[<NUM_LIT:1>], avg])<EOL><DEDENT><DEDENT>return averagedRecordList, historicalValues, total<EOL>

Given a list of anomaly scores return a list of averaged records.
anomalyScores is assumed to be a list of records of the form:
              [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]

Each record in the returned list list contains:
    [datetime, value, averagedScore]

*Note:* we only average the anomaly score.


params = {<EOL>"<STR_LIT:name>": "<STR_LIT>",<EOL>"<STR_LIT>": numpy.mean(sampleData),<EOL>"<STR_LIT>": numpy.var(sampleData),<EOL>}<EOL>if performLowerBoundCheck:<EOL><INDENT>if params["<STR_LIT>"] < <NUM_LIT>:<EOL><INDENT>params["<STR_LIT>"] = <NUM_LIT><EOL><DEDENT>if params["<STR_LIT>"] < <NUM_LIT>:<EOL><INDENT>params["<STR_LIT>"] = <NUM_LIT><EOL><DEDENT><DEDENT>if params["<STR_LIT>"] > <NUM_LIT:0>:<EOL><INDENT>params["<STR_LIT>"] = math.sqrt(params["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>params["<STR_LIT>"] = <NUM_LIT:0><EOL><DEDENT>return params<EOL>

:param sampleData:
:type sampleData: Numpy array.
:param performLowerBoundCheck:
:type performLowerBoundCheck: bool
:returns: A dict containing the parameters of a normal distribution based on
    the ``sampleData``.


if verbosity><NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return {<EOL>"<STR_LIT:name>": "<STR_LIT>",<EOL>"<STR_LIT>": <NUM_LIT:0.5>,<EOL>"<STR_LIT>": <NUM_LIT>,<EOL>"<STR_LIT>": <NUM_LIT>,<EOL>}<EOL>

:param verbosity: integer controlling extent of printouts for debugging
:type verbosity: int
:returns: A distribution that is very broad and makes every anomaly score
    between 0 and 1 pretty likely.


if "<STR_LIT>" not in distributionParams or "<STR_LIT>" not in distributionParams:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if x < distributionParams["<STR_LIT>"]:<EOL><INDENT>xp = <NUM_LIT:2> * distributionParams["<STR_LIT>"] - x<EOL>return tailProbability(xp, distributionParams)<EOL><DEDENT>z = (x - distributionParams["<STR_LIT>"]) / distributionParams["<STR_LIT>"]<EOL>return <NUM_LIT:0.5> * math.erfc(z/<NUM_LIT>)<EOL>

Given the normal distribution specified by the mean and standard deviation
in distributionParams, return the probability of getting samples further
from the mean. For values above the mean, this is the probability of getting
samples > x and for values below the mean, the probability of getting
samples < x. This is the Q-function: the tail probability of the normal distribution.

:param distributionParams: dict with 'mean' and 'stdev' of the distribution


if not isinstance(p, dict):<EOL><INDENT>return False<EOL><DEDENT>if "<STR_LIT>" not in p:<EOL><INDENT>return False<EOL><DEDENT>if "<STR_LIT>" not in p:<EOL><INDENT>return False<EOL><DEDENT>dist = p["<STR_LIT>"]<EOL>if not ("<STR_LIT>" in dist and "<STR_LIT:name>" in dist<EOL>and "<STR_LIT>" in dist and "<STR_LIT>" in dist):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

:returns: ``True`` if ``p`` is a valid estimator params as might be returned
  by ``estimateAnomalyLikelihoods()`` or ``updateAnomalyLikelihoods``,
  ``False`` otherwise.  Just does some basic validation.


if historicWindowSize < estimationSamples:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self._iteration = <NUM_LIT:0><EOL>self._historicalScores = collections.deque(maxlen=historicWindowSize)<EOL>self._distribution = None<EOL>if claLearningPeriod != None:<EOL><INDENT>print("<STR_LIT>")<EOL>self._learningPeriod = claLearningPeriod<EOL><DEDENT>else:<EOL><INDENT>self._learningPeriod = learningPeriod<EOL><DEDENT>self._probationaryPeriod = self._learningPeriod + estimationSamples<EOL>self._reestimationPeriod = reestimationPeriod<EOL>

NOTE: Anomaly likelihood scores are reported at a flat 0.5 for
learningPeriod + estimationSamples iterations.

claLearningPeriod and learningPeriod are specifying the same variable,
although claLearningPeriod is a deprecated name for it.

:param learningPeriod: (claLearningPeriod: deprecated) - (int) the number of
  iterations required for the algorithm to learn the basic patterns in the
  dataset and for the anomaly score to 'settle down'. The default is based
  on empirical observations but in reality this could be larger for more
  complex domains. The downside if this is too large is that real anomalies
  might get ignored and not flagged.

:param estimationSamples: (int) the number of reasonable anomaly scores
  required for the initial estimate of the Gaussian. The default of 100
  records is reasonable - we just need sufficient samples to get a decent
  estimate for the Gaussian. It's unlikely you will need to tune this since
  the Gaussian is re-estimated every 10 iterations by default.

:param historicWindowSize: (int) size of sliding window of historical
  data points to maintain for periodic reestimation of the Gaussian. Note:
  the default of 8640 is based on a month's worth of history at 5-minute
  intervals.

:param reestimationPeriod: (int) how often we re-estimate the Gaussian
  distribution. The ideal is to re-estimate every iteration but this is a
  performance hit. In general the system is not very sensitive to this
  number as long as it is small relative to the total number of records
  processed.


<EOL>return math.log(<NUM_LIT> - likelihood) / -<NUM_LIT><EOL>

Compute a log scale representation of the likelihood value. Since the
likelihood computations return low probabilities that often go into four 9's
or five 9's, a log value is more useful for visualization, thresholding,
etc.


@staticmethod<EOL><INDENT>def _calcSkipRecords(numIngested, windowSize, learningPeriod):<DEDENT>

numShiftedOut = max(<NUM_LIT:0>, numIngested - windowSize)<EOL>return min(numIngested, max(<NUM_LIT:0>, learningPeriod - numShiftedOut))<EOL>

Return the value of skipRecords for passing to estimateAnomalyLikelihoods

        If `windowSize` is very large (bigger than the amount of data) then this
        could just return `learningPeriod`. But when some values have fallen out of
        the historical sliding window of anomaly records, then we have to take those
        into account as well so we return the `learningPeriod` minus the number
        shifted out.

        :param numIngested - (int) number of data points that have been added to the
          sliding window of historical data points.
        :param windowSize - (int) size of sliding window of historical data points.
        :param learningPeriod - (int) the number of iterations required for the
          algorithm to learn the basic patterns in the dataset and for the anomaly
          score to 'settle down'.


<EOL>anomalyLikelihood = object.__new__(cls)<EOL>anomalyLikelihood._iteration = proto.iteration<EOL>anomalyLikelihood._historicalScores = collections.deque(<EOL>maxlen=proto.historicWindowSize)<EOL>for i, score in enumerate(proto.historicalScores):<EOL><INDENT>anomalyLikelihood._historicalScores.append((i, score.value,<EOL>score.anomalyScore))<EOL><DEDENT>if proto.distribution.name: <EOL><INDENT>anomalyLikelihood._distribution = dict()<EOL>anomalyLikelihood._distribution['<STR_LIT>'] = dict()<EOL>anomalyLikelihood._distribution['<STR_LIT>']["<STR_LIT:name>"] = proto.distribution.name<EOL>anomalyLikelihood._distribution['<STR_LIT>']["<STR_LIT>"] = proto.distribution.mean<EOL>anomalyLikelihood._distribution['<STR_LIT>']["<STR_LIT>"] = proto.distribution.variance<EOL>anomalyLikelihood._distribution['<STR_LIT>']["<STR_LIT>"] = proto.distribution.stdev<EOL>anomalyLikelihood._distribution["<STR_LIT>"] = {}<EOL>anomalyLikelihood._distribution["<STR_LIT>"]["<STR_LIT>"] = proto.distribution.movingAverage.windowSize<EOL>anomalyLikelihood._distribution["<STR_LIT>"]["<STR_LIT>"] = []<EOL>for value in proto.distribution.movingAverage.historicalValues:<EOL><INDENT>anomalyLikelihood._distribution["<STR_LIT>"]["<STR_LIT>"].append(value)<EOL><DEDENT>anomalyLikelihood._distribution["<STR_LIT>"]["<STR_LIT>"] = proto.distribution.movingAverage.total<EOL>anomalyLikelihood._distribution["<STR_LIT>"] = []<EOL>for likelihood in proto.distribution.historicalLikelihoods:<EOL><INDENT>anomalyLikelihood._distribution["<STR_LIT>"].append(likelihood)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>anomalyLikelihood._distribution = None<EOL><DEDENT>anomalyLikelihood._probationaryPeriod = proto.probationaryPeriod<EOL>anomalyLikelihood._learningPeriod = proto.learningPeriod<EOL>anomalyLikelihood._reestimationPeriod = proto.reestimationPeriod<EOL>return anomalyLikelihood<EOL>

capnp deserialization method for the anomaly likelihood object

        :param proto: (Object) capnp proto object specified in
                              nupic.regions.anomaly_likelihood.capnp

        :returns: (Object) the deserialized AnomalyLikelihood object


proto.iteration = self._iteration<EOL>pHistScores = proto.init('<STR_LIT>', len(self._historicalScores))<EOL>for i, score in enumerate(list(self._historicalScores)):<EOL><INDENT>_, value, anomalyScore = score<EOL>record = pHistScores[i]<EOL>record.value = float(value)<EOL>record.anomalyScore = float(anomalyScore)<EOL><DEDENT>if self._distribution:<EOL><INDENT>proto.distribution.name = self._distribution["<STR_LIT>"]["<STR_LIT:name>"]<EOL>proto.distribution.mean = float(self._distribution["<STR_LIT>"]["<STR_LIT>"])<EOL>proto.distribution.variance = float(self._distribution["<STR_LIT>"]["<STR_LIT>"])<EOL>proto.distribution.stdev = float(self._distribution["<STR_LIT>"]["<STR_LIT>"])<EOL>proto.distribution.movingAverage.windowSize = float(self._distribution["<STR_LIT>"]["<STR_LIT>"])<EOL>historicalValues = self._distribution["<STR_LIT>"]["<STR_LIT>"]<EOL>pHistValues = proto.distribution.movingAverage.init(<EOL>"<STR_LIT>", len(historicalValues))<EOL>for i, value in enumerate(historicalValues):<EOL><INDENT>pHistValues[i] = float(value)<EOL><DEDENT>proto.distribution.movingAverage.total = float(self._distribution["<STR_LIT>"]["<STR_LIT>"])<EOL>historicalLikelihoods = self._distribution["<STR_LIT>"]<EOL>pHistLikelihoods = proto.distribution.init("<STR_LIT>",<EOL>len(historicalLikelihoods))<EOL>for i, likelihood in enumerate(historicalLikelihoods):<EOL><INDENT>pHistLikelihoods[i] = float(likelihood)<EOL><DEDENT><DEDENT>proto.probationaryPeriod = self._probationaryPeriod<EOL>proto.learningPeriod = self._learningPeriod<EOL>proto.reestimationPeriod = self._reestimationPeriod<EOL>proto.historicWindowSize = self._historicalScores.maxlen<EOL>

capnp serialization method for the anomaly likelihood object

        :param proto: (Object) capnp proto object specified in
                              nupic.regions.anomaly_likelihood.capnp


def anomalyProbability(self, value, anomalyScore, timestamp=None):

if timestamp is None:<EOL><INDENT>timestamp = self._iteration<EOL><DEDENT>dataPoint = (timestamp, value, anomalyScore)<EOL>if self._iteration < self._probationaryPeriod:<EOL><INDENT>likelihood = <NUM_LIT:0.5><EOL><DEDENT>else:<EOL><INDENT>if ( (self._distribution is None) or<EOL>(self._iteration % self._reestimationPeriod == <NUM_LIT:0>) ):<EOL><INDENT>numSkipRecords = self._calcSkipRecords(<EOL>numIngested=self._iteration,<EOL>windowSize=self._historicalScores.maxlen,<EOL>learningPeriod=self._learningPeriod)<EOL>_, _, self._distribution = estimateAnomalyLikelihoods(<EOL>self._historicalScores,<EOL>skipRecords=numSkipRecords)<EOL><DEDENT>likelihoods, _, self._distribution = updateAnomalyLikelihoods(<EOL>[dataPoint],<EOL>self._distribution)<EOL>likelihood = <NUM_LIT:1.0> - likelihoods[<NUM_LIT:0>]<EOL><DEDENT>self._historicalScores.append(dataPoint)<EOL>self._iteration += <NUM_LIT:1><EOL>return likelihood<EOL>

Compute the probability that the current value plus anomaly score represents
an anomaly given the historical distribution of anomaly scores. The closer
the number is to 1, the higher the chance it is an anomaly.

:param value: the current metric ("raw") input value, eg. "orange", or
               '21.2' (deg. Celsius), ...
:param anomalyScore: the current anomaly score
:param timestamp: [optional] timestamp of the ocurrence,
                   default (None) results in using iteration step.
:returns: the anomalyLikelihood for this record.


random.seed(seed)<EOL>numpy.random.seed(seed)<EOL>

Set the random seeds. Helpful to make unit tests repeatable


def addNoise(input, noise=<NUM_LIT:0.1>, doForeground=True, doBackground=True):

if doForeground and doBackground:<EOL><INDENT>return numpy.abs(input -  (numpy.random.random(input.shape) < noise))<EOL><DEDENT>else:<EOL><INDENT>if doForeground:<EOL><INDENT>return numpy.logical_and(input, numpy.random.random(input.shape) > noise)<EOL><DEDENT>if doBackground:<EOL><INDENT>return numpy.logical_or(input, numpy.random.random(input.shape) < noise)<EOL><DEDENT><DEDENT>return input<EOL>

Add noise to the given input.

Parameters:
-----------------------------------------------
input:         the input to add noise to
noise:         how much noise to add
doForeground:  If true, turn off some of the 1 bits in the input
doBackground:  If true, turn on some of the 0 bits in the input


def generateCoincMatrix(nCoinc=<NUM_LIT:10>, length=<NUM_LIT>, activity=<NUM_LIT:50>):

coincMatrix0 = SM32(int(nCoinc), int(length))<EOL>theOnes = numpy.array([<NUM_LIT:1.0>] * activity, dtype=numpy.float32)<EOL>for rowIdx in range(nCoinc):<EOL><INDENT>coinc = numpy.array(random.sample(range(length),<EOL>activity), dtype=numpy.uint32)<EOL>coinc.sort()<EOL>coincMatrix0.setRowFromSparse(rowIdx, coinc, theOnes)<EOL><DEDENT>coincMatrix = SM32(int(nCoinc), int(length))<EOL>coincMatrix.initializeWithFixedNNZR(activity)<EOL>return coincMatrix0<EOL>

Generate a coincidence matrix. This is used to generate random inputs to the
temporal learner and to compare the predicted output against.

It generates a matrix of nCoinc rows, each row has length 'length' and has
a total of 'activity' bits on.

Parameters:
-----------------------------------------------
nCoinc:        the number of rows to generate
length:        the length of each row
activity:      the number of ones to put into each row.


def generateVectors(numVectors=<NUM_LIT:100>, length=<NUM_LIT>, activity=<NUM_LIT:50>):

vectors = []<EOL>coinc = numpy.zeros(length, dtype='<STR_LIT>')<EOL>indexList = list(range(length))<EOL>for i in range(numVectors):<EOL><INDENT>coinc[:] = <NUM_LIT:0><EOL>coinc[random.sample(indexList, activity)] = <NUM_LIT:1><EOL>vectors.append(coinc.copy())<EOL><DEDENT>return vectors<EOL>

Generate a list of random sparse distributed vectors.  This is used to generate
training vectors to the spatial or temporal learner and to compare the predicted
output against.

It generates a list of 'numVectors' elements, each element has length 'length'
and has a total of 'activity' bits on.

Parameters:
-----------------------------------------------
numVectors:    the number of vectors to generate
length:        the length of each row
activity:      the number of ones to put into each row.


def generateSimpleSequences(nCoinc=<NUM_LIT:10>, seqLength=[<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>], nSeq=<NUM_LIT:100>):

coincList = list(range(nCoinc))<EOL>seqList  = []<EOL>for i in range(nSeq):<EOL><INDENT>if max(seqLength) <= nCoinc:<EOL><INDENT>seqList.append(random.sample(coincList, random.choice(seqLength)))<EOL><DEDENT>else:<EOL><INDENT>len = random.choice(seqLength)<EOL>seq = []<EOL>for x in range(len):<EOL><INDENT>seq.append(random.choice(coincList))<EOL><DEDENT>seqList.append(seq)<EOL><DEDENT><DEDENT>return seqList<EOL>

Generate a set of simple sequences. The elements of the sequences will be
integers from 0 to 'nCoinc'-1. The length of each sequence will be
randomly chosen from the 'seqLength' list.

Parameters:
-----------------------------------------------
nCoinc:      the number of elements available to use in the sequences
seqLength:   a list of possible sequence lengths. The length of each
             sequence will be randomly chosen from here.
nSeq:        The number of sequences to generate

retval:      a list of sequences. Each sequence is itself a list
             containing the coincidence indices for that sequence.


def generateHubSequences(nCoinc=<NUM_LIT:10>, hubs = [<NUM_LIT:2>,<NUM_LIT:6>], seqLength=[<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>], nSeq=<NUM_LIT:100>):

coincList = list(range(nCoinc))<EOL>for hub in hubs:<EOL><INDENT>coincList.remove(hub)<EOL><DEDENT>seqList = []<EOL>for i in range(nSeq):<EOL><INDENT>length = random.choice(seqLength)-<NUM_LIT:1><EOL>seq = random.sample(coincList,length)<EOL>seq.insert(length//<NUM_LIT:2>, random.choice(hubs))<EOL>seqList.append(seq)<EOL><DEDENT>return seqList<EOL>

Generate a set of hub sequences. These are sequences which contain a hub
element in the middle. The elements of the sequences will be integers
from 0 to 'nCoinc'-1. The hub elements will only appear in the middle of
each sequence. The length of each sequence will be randomly chosen from the
'seqLength' list.

Parameters:
-----------------------------------------------
nCoinc:        the number of elements available to use in the sequences
hubs:          which of the elements will be used as hubs.
seqLength:     a list of possible sequence lengths. The length of each
                      sequence will be randomly chosen from here.
nSeq:          The number of sequences to generate

retval:        a list of sequences. Each sequence is itself a list
              containing the coincidence indices for that sequence.


<EOL>patterns = generateCoincMatrix(nCoinc=nPatterns, length=patternLen,<EOL>activity=patternActivity)<EOL>similarity = []<EOL>for i in range(nPatterns):<EOL><INDENT>similarity.append(patterns.rightVecProd(patterns.getRow(i)))<EOL><DEDENT>similarity = numpy.array(similarity, dtype='<STR_LIT>')<EOL>print(similarity)<EOL>seqList1 =  generateSimpleSequences(nCoinc=nPatterns, seqLength=seqLength,<EOL>nSeq=nSequences)<EOL>seqList2 = copy.deepcopy(seqList1)<EOL>for i in range(<NUM_LIT:0>,len(seqList2)):<EOL><INDENT>seqList2[i][<NUM_LIT:0>] = random.randint(<NUM_LIT:0>,nPatterns-<NUM_LIT:1>)<EOL><DEDENT>return (seqList1, seqList2, patterns)<EOL>

Generate two sets of sequences. The first set of sequences is used to train
the sequence learner till it fills up capacity. The second set is then used
to further train the system to test its generalization capability using the
one step look back idea. The second set of sequences are generated by modifying
the first set

Parameters:
-----------------------------------------------
nPatterns:        the number of patterns to use in the sequences.
patternLen:       The number of elements in each pattern
patternActivity:  The number of elements that should be active in
                      each pattern
seqLength:        a list of possible sequence lengths. The length of each
                      sequence will be randomly chosen from here.
nSequences: The number of simple sequences in the first set

retval:           (seqList1, seqList2, patterns)
                  seqList1, seqList2: a list of sequences. Each sequence is itself a list
                                containing the input pattern indices for that sequence.
                  patterns: the input patterns used in the seqList.


def generateSimpleCoincMatrix(nCoinc=<NUM_LIT:10>, length=<NUM_LIT>, activity=<NUM_LIT:50>):

assert nCoinc*activity<=length, "<STR_LIT>"<EOL>coincMatrix = SM32(<NUM_LIT:0>, length)<EOL>coinc = numpy.zeros(length, dtype='<STR_LIT>')<EOL>for i in range(nCoinc):<EOL><INDENT>coinc[:] = <NUM_LIT:0><EOL>coinc[i*activity:(i+<NUM_LIT:1>)*activity] = <NUM_LIT:1><EOL>coincMatrix.addRow(coinc)<EOL><DEDENT>return coincMatrix<EOL>

Generate a non overlapping coincidence matrix. This is used to generate random
inputs to the temporal learner and to compare the predicted output against.

It generates a matrix of nCoinc rows, each row has length 'length' and has
a total of 'activity' bits on.

Parameters:
-----------------------------------------------
nCoinc:        the number of rows to generate
length:        the length of each row
activity:      the number of ones to put into each row.


def generateSequences(nPatterns=<NUM_LIT:10>, patternLen=<NUM_LIT>, patternActivity=<NUM_LIT:50>,<EOL>hubs=[<NUM_LIT:2>,<NUM_LIT:6>],  seqLength=[<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>],<EOL>nSimpleSequences=<NUM_LIT:50>,  nHubSequences=<NUM_LIT:50>):

<EOL>patterns = generateCoincMatrix(nCoinc=nPatterns, length=patternLen,<EOL>activity=patternActivity)<EOL>seqList =  generateSimpleSequences(nCoinc=nPatterns, seqLength=seqLength,<EOL>nSeq=nSimpleSequences) +generateHubSequences(nCoinc=nPatterns, hubs=hubs, seqLength=seqLength,<EOL>nSeq=nHubSequences)<EOL>return (seqList, patterns)<EOL>

Generate a set of simple and hub sequences. A simple sequence contains
a randomly chosen set of elements from 0 to 'nCoinc-1'. A hub sequence
always contains a hub element in the middle of it.

Parameters:
-----------------------------------------------
nPatterns:        the number of patterns to use in the sequences.
patternLen:       The number of elements in each pattern
patternActivity:  The number of elements that should be active in
                      each pattern
hubs:             which of the elements will be used as hubs.
seqLength:        a list of possible sequence lengths. The length of each
                      sequence will be randomly chosen from here.
nSimpleSequences: The number of simple sequences to generate
nHubSequences:    The number of hub sequences to generate

retval:           (seqList, patterns)
                  seqList: a list of sequences. Each sequence is itself a list
                                containing the input pattern indices for that sequence.
                  patterns: the input patterns used in the seqList.


<EOL>l1SeqList = generateSimpleSequences(nCoinc=nL1Patterns, seqLength=l1SeqLength,<EOL>nSeq=nL1SimpleSequences) +generateHubSequences(nCoinc=nL1Patterns, hubs=l1Hubs,<EOL>seqLength=l1SeqLength, nSeq=nL1HubSequences)<EOL>spOutput = generateSlowSPOutput(seqListBelow = l1SeqList,<EOL>poolingTimeBelow=l1Pooling, outputWidth=patternLen,<EOL>activity=patternActivity, perfectStability=perfectStability,<EOL>spHysteresisFactor=spHysteresisFactor)<EOL>outSeq = None<EOL>outSeqList = []<EOL>outPatterns = SM32(<NUM_LIT:0>, patternLen)<EOL>for pattern in spOutput:<EOL><INDENT>if pattern.sum() == <NUM_LIT:0>:<EOL><INDENT>if outSeq is not None:<EOL><INDENT>outSeqList.append(outSeq)<EOL><DEDENT>outSeq = []<EOL>continue<EOL><DEDENT>patternIdx = None<EOL>if outPatterns.nRows() > <NUM_LIT:0>:<EOL><INDENT>matches = outPatterns.rightVecSumAtNZ(pattern)<EOL>outCoinc = matches.argmax().astype('<STR_LIT>')<EOL>numOnes = pattern.sum()<EOL>if matches[outCoinc] == numOnesand outPatterns.getRow(int(outCoinc)).sum() == numOnes:<EOL><INDENT>patternIdx = outCoinc<EOL><DEDENT><DEDENT>if patternIdx is None:<EOL><INDENT>outPatterns.addRow(pattern)<EOL>patternIdx = outPatterns.nRows() - <NUM_LIT:1><EOL><DEDENT>outSeq.append(patternIdx)<EOL><DEDENT>if outSeq is not None:<EOL><INDENT>outSeqList.append(outSeq)<EOL><DEDENT>return (outSeqList, outPatterns)<EOL>

Generate the simulated output from a spatial pooler that's sitting
on top of another spatial pooler / temporal memory pair.  The average on-time
of the outputs from the simulated TM is given by the l1Pooling argument.

In this routine, L1 refers to the first spatial and temporal memory and L2
refers to the spatial pooler above that.

Parameters:
-----------------------------------------------
nL1Patterns:          the number of patterns to use in the L1 sequences.
l1Hubs:               which of the elements will be used as hubs.
l1SeqLength:          a list of possible sequence lengths. The length of each
                      sequence will be randomly chosen from here.
nL1SimpleSequences:   The number of simple sequences to generate for L1
nL1HubSequences:      The number of hub sequences to generate for L1
l1Pooling:            The number of time steps to pool over in the L1 temporal
                        pooler
perfectStability:     If true, then the input patterns represented by the
                      sequences generated will have perfect stability over
                      l1Pooling time steps. This is the best case ideal input
                      to a TM. In actual situations, with an actual SP
                      providing input, the stability will always be less than
                      this.
spHystereisFactor:    The hysteresisFactor to use in the L2 spatial pooler.
                      Only used when perfectStability is  False
patternLen:           The number of elements in each pattern output by L2
patternActivity:      The number of elements that should be active in
                      each pattern

@retval:              (seqList, patterns)
                      seqList: a list of sequences output from L2. Each sequence is
                          itself a list containing the input pattern indices for that
                          sequence.
                      patterns: the input patterns used in the L2 seqList.


totalLen = <NUM_LIT:0><EOL>for seq in seqList:<EOL><INDENT>totalLen += len(seq)<EOL><DEDENT>vectors = numpy.zeros((totalLen, patternMatrix.shape[<NUM_LIT:1>]), dtype='<STR_LIT:bool>')<EOL>vecOffset = <NUM_LIT:0><EOL>for seq in seqList:<EOL><INDENT>seq = numpy.array(seq, dtype='<STR_LIT>')<EOL>for idx,coinc in enumerate(seq):<EOL><INDENT>vectors[vecOffset] = patternMatrix.getRow(int(coinc))<EOL>vecOffset += <NUM_LIT:1><EOL><DEDENT><DEDENT>return vectors<EOL>

Convert a list of sequences of pattern indices, and a pattern lookup table
    into a an array of patterns

Parameters:
-----------------------------------------------
seq:            the sequence, given as indices into the patternMatrix
patternMatrix:  a SparseMatrix contaning the possible patterns used in
                        the sequence.


result = True<EOL>for param in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>if getattr(tp1, param) != getattr(tp2,param):<EOL><INDENT>print(param,"<STR_LIT>")<EOL>print(getattr(tp1, param), "<STR_LIT>", getattr(tp2,param))<EOL>result = False<EOL><DEDENT><DEDENT>return result<EOL>

Given two TM instances, see if any parameters are different.


for s in synapses:<EOL><INDENT>if (s[<NUM_LIT:0>]==syn[<NUM_LIT:0>]) and (s[<NUM_LIT:1>]==syn[<NUM_LIT:1>]) and (abs(s[<NUM_LIT:2>]-syn[<NUM_LIT:2>]) <= <NUM_LIT>):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Given a synapse and a list of synapses, check whether this synapse
    exist in the list.  A synapse is represented as [col, cell, permanence].
    A synapse matches if col and cell are identical and the permanence value is
    within 0.001.


result = True<EOL>for field in [<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:4>, <NUM_LIT:5>, <NUM_LIT:6>]:<EOL><INDENT>if abs(seg1[<NUM_LIT:0>][field] - seg2[<NUM_LIT:0>][field]) > <NUM_LIT>:<EOL><INDENT>result = False<EOL><DEDENT><DEDENT>if len(seg1[<NUM_LIT:1>:]) != len(seg2[<NUM_LIT:1>:]):<EOL><INDENT>result = False<EOL><DEDENT>for syn in seg2[<NUM_LIT:1>:]:<EOL><INDENT>if syn[<NUM_LIT:2>] <= <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>result = False<EOL><DEDENT><DEDENT>if result == True:<EOL><INDENT>for syn in seg1[<NUM_LIT:1>:]:<EOL><INDENT>if syn[<NUM_LIT:2>] <= <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>result = False<EOL><DEDENT>res = sameSynapse(syn, seg2[<NUM_LIT:1>:])<EOL>if res == False:<EOL><INDENT>result = False<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>

Return True if seg1 and seg2 are identical, ignoring order of synapses


def tmDiff(tm1, tm2, verbosity = <NUM_LIT:0>, relaxSegmentTests =True):

<EOL>if sameTMParams(tm1, tm2) == False:<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>result = True<EOL>if (tm1.activeState['<STR_LIT:t>'] != tm2.activeState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.activeState['<STR_LIT:t>'] != tm2.activeState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if (tm1.predictedState['<STR_LIT:t>'] - tm2.predictedState['<STR_LIT:t>']).any():<EOL><INDENT>print('<STR_LIT>', numpy.where(tm1.predictedState['<STR_LIT:t>'] != tm2.predictedState['<STR_LIT:t>']))<EOL>result = False<EOL><DEDENT>if tm1.getNumSegments() != tm2.getNumSegments():<EOL><INDENT>print("<STR_LIT>", tm1.getNumSegments(), tm2.getNumSegments())<EOL>result = False<EOL><DEDENT>if tm1.getNumSynapses() != tm2.getNumSynapses():<EOL><INDENT>print("<STR_LIT>", tm1.getNumSynapses(), tm2.getNumSynapses())<EOL>tm1.printCells()<EOL>tm2.printCells()<EOL>result = False<EOL><DEDENT>for c in range(tm1.numberOfCols):<EOL><INDENT>for i in range(tm2.cellsPerColumn):<EOL><INDENT>if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):<EOL><INDENT>print("<STR_LIT>",c,i, end='<STR_LIT:U+0020>')<EOL>print(tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i))<EOL>result = False<EOL><DEDENT><DEDENT><DEDENT>if result == True and not relaxSegmentTests:<EOL><INDENT>for c in range(tm1.numberOfCols):<EOL><INDENT>for i in range(tm2.cellsPerColumn):<EOL><INDENT>nSegs = tm1.getNumSegmentsInCell(c, i)<EOL>for segIdx in range(nSegs):<EOL><INDENT>tm1seg = tm1.getSegmentOnCell(c, i, segIdx)<EOL>res = False<EOL>for tm2segIdx in range(nSegs):<EOL><INDENT>tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)<EOL>if sameSegment(tm1seg, tm2seg) == True:<EOL><INDENT>res = True<EOL>break<EOL><DEDENT><DEDENT>if res == False:<EOL><INDENT>print("<STR_LIT>",c,i)<EOL>if verbosity >= <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>tm1.printCell(c, i)<EOL>print("<STR_LIT>")<EOL>tm2.printCell(c, i)<EOL><DEDENT>result = False<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if result == True and (verbosity > <NUM_LIT:1>):<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return result<EOL>

Given two TM instances, list the difference between them and returns False
if there is a difference. This function checks the major parameters. If this
passes (and checkLearn is true) it checks the number of segments on
each cell. If this passes, checks each synapse on each segment.
When comparing C++ and Py, the segments are usually in different orders in the
cells. tmDiff ignores segment order when comparing TM's.