self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(self._hsObj._jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if self._priorStateJSON is None:<EOL><INDENT>swarms = dict()<EOL>if self._hsObj._fixedFields is not None:<EOL><INDENT>print(self._hsObj._fixedFields)<EOL>encoderSet = []<EOL>for field in self._hsObj._fixedFields:<EOL><INDENT>if field =='<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>encoderName = self.getEncoderKeyFromName(field)<EOL>assert encoderName in self._hsObj._encoderNames, "<STR_LIT>""<STR_LIT>""<STR_LIT>" % (field)<EOL>encoderSet.append(encoderName)<EOL><DEDENT>encoderSet.sort()<EOL>swarms['<STR_LIT:.>'.join(encoderSet)] = {<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>}<EOL><DEDENT>elif self._hsObj._searchType == HsSearchType.temporal:<EOL><INDENT>for encoderName in self._hsObj._encoderNames:<EOL><INDENT>swarms[encoderName] = {<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>}<EOL><DEDENT><DEDENT>elif self._hsObj._searchType == HsSearchType.classification:<EOL><INDENT>for encoderName in self._hsObj._encoderNames:<EOL><INDENT>if encoderName == self._hsObj._predictedFieldEncoder:<EOL><INDENT>continue<EOL><DEDENT>swarms[encoderName] = {<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>}<EOL><DEDENT><DEDENT>elif self._hsObj._searchType == HsSearchType.legacyTemporal:<EOL><INDENT>swarms[self._hsObj._predictedFieldEncoder] = {<EOL>'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>}<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(self._hsObj._searchType))<EOL><DEDENT>self._state = dict(<EOL>lastUpdateTime = time.time(),<EOL>lastGoodSprint = None,<EOL>searchOver = False,<EOL>activeSwarms = list(swarms.keys()),<EOL>swarms = swarms,<EOL>sprints = [{'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None}],<EOL>blackListedEncoders = [],<EOL>)<EOL>self._hsObj._cjDAO.jobSetFieldIfEqual(<EOL>self._hsObj._jobID, '<STR_LIT>', json.dumps(self._state), None)<EOL>self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(<EOL>self._hsObj._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>assert (self._priorStateJSON is not None)<EOL><DEDENT>self._state = json.loads(self._priorStateJSON)<EOL>self._dirty = False<EOL>

Set our state to that obtained from the engWorkerState field of the
        job record.

        Parameters:
        ---------------------------------------------------------------------
        stateJSON:    JSON encoded state from job record


<EOL>if not self._dirty:<EOL><INDENT>return True<EOL><DEDENT>self._state['<STR_LIT>'] = time.time()<EOL>newStateJSON = json.dumps(self._state)<EOL>success = self._hsObj._cjDAO.jobSetFieldIfEqual(self._hsObj._jobID,<EOL>'<STR_LIT>', str(newStateJSON), str(self._priorStateJSON))<EOL>if success:<EOL><INDENT>self.logger.debug("<STR_LIT>" %(pprint.pformat(self._state, indent=<NUM_LIT:4>)))<EOL>self._priorStateJSON = newStateJSON<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug("<STR_LIT>" %(pprint.pformat(self._state, indent=<NUM_LIT:4>)))<EOL>self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(self._hsObj._jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>self._state =  json.loads(self._priorStateJSON)<EOL>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (pprint.pformat(self._state, indent=<NUM_LIT:4>)))<EOL><DEDENT>return success<EOL>

Update the state in the job record with our local changes (if any).
        If we don't have the latest state in our priorStateJSON, then re-load
        in the latest state and return False. If we were successful writing out
        our changes, return True

        Parameters:
        ---------------------------------------------------------------------
        retval:    True if we were successful writing out our changes
                   False if our priorState is not the latest that was in the DB.
                   In this case, we will re-load our state from the DB


return key.split('<STR_LIT:|>')[-<NUM_LIT:1>]<EOL>

Given an encoder dictionary key, get the encoder name.

        Encoders are a sub-dict within model params, and in HSv2, their key
        is structured like this for example:
           'modelParams|sensorParams|encoders|home_winloss'

        The encoderName is the last word in the | separated key name


return '<STR_LIT>' % (name)<EOL>

Given an encoder name, get the key.

        Encoders are a sub-dict within model params, and in HSv2, their key
        is structured like this for example:
           'modelParams|sensorParams|encoders|home_winloss'

        The encoderName is the last word in the | separated key name


<EOL>if self._hsObj._fixedFields is not None:<EOL><INDENT>return dict(), dict()<EOL><DEDENT>predictedEncoderName = self._hsObj._predictedFieldEncoder<EOL>fieldScores = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>encodersUsed = swarmId.split('<STR_LIT:.>')<EOL>if len(encodersUsed) != <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>field = self.getEncoderNameFromKey(encodersUsed[<NUM_LIT:0>])<EOL>bestScore = info['<STR_LIT>']<EOL>if bestScore is None:<EOL><INDENT>(_modelId, bestScore) =self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)<EOL><DEDENT>fieldScores.append((bestScore, field))<EOL><DEDENT>if self._hsObj._searchType == HsSearchType.legacyTemporal:<EOL><INDENT>assert(len(fieldScores)==<NUM_LIT:1>)<EOL>(baseErrScore, baseField) = fieldScores[<NUM_LIT:0>]<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>encodersUsed = swarmId.split('<STR_LIT:.>')<EOL>if len(encodersUsed) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>fields = [self.getEncoderNameFromKey(name) for name in encodersUsed]<EOL>fields.remove(baseField)<EOL>fieldScores.append((info['<STR_LIT>'], fields[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fieldScores.sort(reverse=True)<EOL>if self._hsObj._maxBranching > <NUM_LIT:0>and len(fieldScores) > self._hsObj._maxBranching:<EOL><INDENT>baseErrScore = fieldScores[-self._hsObj._maxBranching-<NUM_LIT:1>][<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>baseErrScore = fieldScores[<NUM_LIT:0>][<NUM_LIT:0>]<EOL><DEDENT><DEDENT>pctFieldContributionsDict = dict()<EOL>absFieldContributionsDict = dict()<EOL>if baseErrScore is not None:<EOL><INDENT>if abs(baseErrScore) < <NUM_LIT>:<EOL><INDENT>baseErrScore = <NUM_LIT><EOL><DEDENT>for (errScore, field) in fieldScores:<EOL><INDENT>if errScore is not None:<EOL><INDENT>pctBetter = (baseErrScore - errScore) * <NUM_LIT> / baseErrScore<EOL><DEDENT>else:<EOL><INDENT>pctBetter = <NUM_LIT:0.0><EOL>errScore = baseErrScore   <EOL><DEDENT>pctFieldContributionsDict[field] = pctBetter<EOL>absFieldContributionsDict[field] = baseErrScore - errScore<EOL><DEDENT><DEDENT>self.logger.debug("<STR_LIT>" % (pctFieldContributionsDict))<EOL>return pctFieldContributionsDict, absFieldContributionsDict<EOL>

Return the field contributions statistics.

        Parameters:
        ---------------------------------------------------------------------
        retval:   Dictionary where the keys are the field names and the values
                    are how much each field contributed to the best score.


swarmIds = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>if info['<STR_LIT>'] == sprintIdx:<EOL><INDENT>swarmIds.append(swarmId)<EOL><DEDENT><DEDENT>return swarmIds<EOL>

Return the list of all swarms in the given sprint.

        Parameters:
        ---------------------------------------------------------------------
        retval:   list of active swarm Ids in the given sprint


swarmIds = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>if sprintIdx is not None and info['<STR_LIT>'] != sprintIdx:<EOL><INDENT>continue<EOL><DEDENT>if info['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>swarmIds.append(swarmId)<EOL><DEDENT><DEDENT>return swarmIds<EOL>

Return the list of active swarms in the given sprint. These are swarms
        which still need new particles created in them.

        Parameters:
        ---------------------------------------------------------------------
        sprintIdx:    which sprint to query. If None, get active swarms from all
                          sprints
        retval:   list of active swarm Ids in the given sprint


swarmIds = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>if info['<STR_LIT>'] == sprintIdx and info['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>swarmIds.append(swarmId)<EOL><DEDENT><DEDENT>return swarmIds<EOL>

Return the list of swarms in the given sprint that were not killed.
        This is called when we are trying to figure out which encoders to carry
        forward to the next sprint. We don't want to carry forward encoder
        combintations which were obviously bad (in killed swarms).

        Parameters:
        ---------------------------------------------------------------------
        retval:   list of active swarm Ids in the given sprint


swarmIds = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>if info['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>swarmIds.append(swarmId)<EOL><DEDENT><DEDENT>return swarmIds<EOL>


swarmInfo = self._state['<STR_LIT>'][swarmId]<EOL>return (swarmInfo['<STR_LIT>'],<EOL>swarmInfo['<STR_LIT>'])<EOL>

Return the best model ID and it's errScore from the given swarm.
        If the swarm has not completed yet, the bestModelID will be None.

        Parameters:
        ---------------------------------------------------------------------
        retval:   (modelId, errScore)


sprintInfo = self._state['<STR_LIT>'][sprintIdx]<EOL>return (sprintInfo['<STR_LIT>'],<EOL>sprintInfo['<STR_LIT>'])<EOL>

Return the best model ID and it's errScore from the given sprint.
        If the sprint has not completed yet, the bestModelID will be None.

        Parameters:
        ---------------------------------------------------------------------
        retval:   (modelId, errScore)


<EOL>swarms = self.getAllSwarms(sprintIdx)<EOL>bestModelId = None<EOL>bestErrScore = numpy.inf<EOL>for swarmId in swarms:<EOL><INDENT>(modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)<EOL>if errScore < bestErrScore:<EOL><INDENT>bestModelId = modelId<EOL>bestErrScore = errScore<EOL><DEDENT><DEDENT>return (bestModelId, bestErrScore)<EOL>

Return the best model ID and it's errScore from the given sprint,
        which may still be in progress. This returns the best score from all models
        in the sprint which have matured so far.

        Parameters:
        ---------------------------------------------------------------------
        retval:   (modelId, errScore)


def setSwarmState(self, swarmId, newStatus):

assert (newStatus in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>swarmInfo = self._state['<STR_LIT>'][swarmId]<EOL>if swarmInfo['<STR_LIT:status>'] == newStatus:<EOL><INDENT>return<EOL><DEDENT>if swarmInfo['<STR_LIT:status>'] == '<STR_LIT>' and newStatus == '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>self._dirty = True<EOL>swarmInfo['<STR_LIT:status>'] = newStatus<EOL>if newStatus == '<STR_LIT>':<EOL><INDENT>(modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)<EOL>swarmInfo['<STR_LIT>'] = modelId<EOL>swarmInfo['<STR_LIT>'] = errScore<EOL><DEDENT>if newStatus != '<STR_LIT>' and swarmId in self._state['<STR_LIT>']:<EOL><INDENT>self._state['<STR_LIT>'].remove(swarmId)<EOL><DEDENT>if newStatus=='<STR_LIT>':<EOL><INDENT>self._hsObj.killSwarmParticles(swarmId)<EOL><DEDENT>sprintIdx = swarmInfo['<STR_LIT>']<EOL>self.isSprintActive(sprintIdx)<EOL>sprintInfo = self._state['<STR_LIT>'][sprintIdx]<EOL>statusCounts = dict(active=<NUM_LIT:0>, completing=<NUM_LIT:0>, completed=<NUM_LIT:0>, killed=<NUM_LIT:0>)<EOL>bestModelIds = []<EOL>bestErrScores = []<EOL>for info in self._state['<STR_LIT>'].values():<EOL><INDENT>if info['<STR_LIT>'] != sprintIdx:<EOL><INDENT>continue<EOL><DEDENT>statusCounts[info['<STR_LIT:status>']] += <NUM_LIT:1><EOL>if info['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>bestModelIds.append(info['<STR_LIT>'])<EOL>bestErrScores.append(info['<STR_LIT>'])<EOL><DEDENT><DEDENT>if statusCounts['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>sprintStatus = '<STR_LIT>'<EOL><DEDENT>elif statusCounts['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>sprintStatus = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>sprintStatus = '<STR_LIT>'<EOL><DEDENT>sprintInfo['<STR_LIT:status>'] = sprintStatus<EOL>if sprintStatus == '<STR_LIT>':<EOL><INDENT>if len(bestErrScores) > <NUM_LIT:0>:<EOL><INDENT>whichIdx = numpy.array(bestErrScores).argmin()<EOL>sprintInfo['<STR_LIT>'] = bestModelIds[whichIdx]<EOL>sprintInfo['<STR_LIT>'] = bestErrScores[whichIdx]<EOL><DEDENT>else:<EOL><INDENT>sprintInfo['<STR_LIT>'] = <NUM_LIT:0><EOL>sprintInfo['<STR_LIT>'] = numpy.inf<EOL><DEDENT>bestPrior = numpy.inf<EOL>for idx in range(sprintIdx):<EOL><INDENT>if self._state['<STR_LIT>'][idx]['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>(_, errScore) = self.bestModelInCompletedSprint(idx)<EOL>if errScore is None:<EOL><INDENT>errScore = numpy.inf<EOL><DEDENT><DEDENT>else:<EOL><INDENT>errScore = numpy.inf<EOL><DEDENT>if errScore < bestPrior:<EOL><INDENT>bestPrior = errScore<EOL><DEDENT><DEDENT>if sprintInfo['<STR_LIT>'] >= bestPrior:<EOL><INDENT>self._state['<STR_LIT>'] = sprintIdx-<NUM_LIT:1><EOL><DEDENT>if self._state['<STR_LIT>'] is not Noneand not self.anyGoodSprintsActive():<EOL><INDENT>self._state['<STR_LIT>'] = True<EOL><DEDENT><DEDENT>

Change the given swarm's state to 'newState'. If 'newState' is
        'completed', then bestModelId and bestErrScore must be provided.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:      swarm Id
        newStatus:    new status, either 'active', 'completing', 'completed', or
                        'killed'


if self._state['<STR_LIT>'] is not None:<EOL><INDENT>goodSprints = self._state['<STR_LIT>'][<NUM_LIT:0>:self._state['<STR_LIT>']+<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>goodSprints = self._state['<STR_LIT>']<EOL><DEDENT>for sprint in goodSprints:<EOL><INDENT>if sprint['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>anyActiveSprints = True<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>anyActiveSprints = False<EOL><DEDENT>return anyActiveSprints<EOL>

Return True if there are any more good sprints still being explored.
        A 'good' sprint is one that is earlier than where we detected an increase
        in error from sprint to subsequent sprint.


numExistingSprints = len(self._state['<STR_LIT>'])<EOL>if sprintIdx >= numExistingSprints:<EOL><INDENT>return False<EOL><DEDENT>return (self._state['<STR_LIT>'][sprintIdx]['<STR_LIT:status>'] == '<STR_LIT>')<EOL>

Return True if the given sprint has completed.


<EOL>numExistingSprints = len(self._state['<STR_LIT>'])<EOL>if self._hsObj._searchType == HsSearchType.legacyTemporal:<EOL><INDENT>if numExistingSprints <= <NUM_LIT:2>:<EOL><INDENT>return<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if numExistingSprints <= <NUM_LIT:1>:<EOL><INDENT>return<EOL><DEDENT><DEDENT>completedSwarms = self.getCompletedSwarms()<EOL>completedSwarms = [(swarm, self._state["<STR_LIT>"][swarm],<EOL>self._state["<STR_LIT>"][swarm]["<STR_LIT>"])for swarm in completedSwarms]<EOL>completedMatrix = [[] for i in range(numExistingSprints)]<EOL>for swarm in completedSwarms:<EOL><INDENT>completedMatrix[swarm[<NUM_LIT:1>]["<STR_LIT>"]].append(swarm)<EOL><DEDENT>for sprint in completedMatrix:<EOL><INDENT>sprint.sort(key=itemgetter(<NUM_LIT:2>))<EOL><DEDENT>activeSwarms = self.getActiveSwarms()<EOL>activeSwarms.extend(self.getCompletingSwarms())<EOL>activeSwarms = [(swarm, self._state["<STR_LIT>"][swarm],<EOL>self._state["<STR_LIT>"][swarm]["<STR_LIT>"])for swarm in activeSwarms]<EOL>activeMatrix = [[] for i in range(numExistingSprints)]<EOL>for swarm in activeSwarms:<EOL><INDENT>activeMatrix[swarm[<NUM_LIT:1>]["<STR_LIT>"]].append(swarm)<EOL><DEDENT>for sprint in activeMatrix:<EOL><INDENT>sprint.sort(key=itemgetter(<NUM_LIT:2>))<EOL><DEDENT>toKill = []<EOL>for i in range(<NUM_LIT:1>, numExistingSprints):<EOL><INDENT>for swarm in activeMatrix[i]:<EOL><INDENT>curSwarmEncoders = swarm[<NUM_LIT:0>].split("<STR_LIT:.>")<EOL>if(len(activeMatrix[i-<NUM_LIT:1>])==<NUM_LIT:0>):<EOL><INDENT>if i==<NUM_LIT:2> and (self._hsObj._tryAll3FieldCombinations orself._hsObj._tryAll3FieldCombinationsWTimestamps):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>bestInPrevious = completedMatrix[i-<NUM_LIT:1>][<NUM_LIT:0>]<EOL>bestEncoders = bestInPrevious[<NUM_LIT:0>].split('<STR_LIT:.>')<EOL>for encoder in bestEncoders:<EOL><INDENT>if not encoder in curSwarmEncoders:<EOL><INDENT>toKill.append(swarm)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if len(toKill) > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" + str(toKill))<EOL><DEDENT>for swarm in toKill:<EOL><INDENT>self.setSwarmState(swarm[<NUM_LIT:0>], "<STR_LIT>")<EOL><DEDENT>return<EOL>

See if we can kill off some speculative swarms. If an earlier sprint
        has finally completed, we can now tell which fields should *really* be present
        in the sprints we've already started due to speculation, and kill off the
        swarms that should not have been included.


while True:<EOL><INDENT>numExistingSprints = len(self._state['<STR_LIT>'])<EOL>if sprintIdx <= numExistingSprints-<NUM_LIT:1>:<EOL><INDENT>if not self._hsObj._speculativeParticles:<EOL><INDENT>active = (self._state['<STR_LIT>'][sprintIdx]['<STR_LIT:status>'] == '<STR_LIT>')<EOL>return (active, False)<EOL><DEDENT>else:<EOL><INDENT>active = (self._state['<STR_LIT>'][sprintIdx]['<STR_LIT:status>'] == '<STR_LIT>')<EOL>if not active:<EOL><INDENT>return (active, False)<EOL><DEDENT>activeSwarmIds = self.getActiveSwarms(sprintIdx)<EOL>swarmSizes = [self._hsObj._resultsDB.getParticleInfos(swarmId,<EOL>matured=False)[<NUM_LIT:0>] for swarmId in activeSwarmIds]<EOL>notFullSwarms = [len(swarm) for swarm in swarmSizesif len(swarm) < self._hsObj._minParticlesPerSwarm]<EOL>if len(notFullSwarms) > <NUM_LIT:0>:<EOL><INDENT>return (True, False)<EOL><DEDENT><DEDENT><DEDENT>if self._state['<STR_LIT>'] is not None:<EOL><INDENT>return (False, True)<EOL><DEDENT>if self._hsObj._fixedFields is not None:<EOL><INDENT>return (False, True)<EOL><DEDENT>if sprintIdx > <NUM_LIT:0>and self._state['<STR_LIT>'][sprintIdx-<NUM_LIT:1>]['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>(bestModelId, _) = self.bestModelInCompletedSprint(sprintIdx-<NUM_LIT:1>)<EOL>(particleState, _, _, _, _) = self._hsObj._resultsDB.getParticleInfo(<EOL>bestModelId)<EOL>bestSwarmId = particleState['<STR_LIT>']<EOL>baseEncoderSets = [bestSwarmId.split('<STR_LIT:.>')]<EOL><DEDENT>else:<EOL><INDENT>bestSwarmId = None<EOL>particleState = None<EOL>baseEncoderSets = []<EOL>for swarmId in self.getNonKilledSwarms(sprintIdx-<NUM_LIT:1>):<EOL><INDENT>baseEncoderSets.append(swarmId.split('<STR_LIT:.>'))<EOL><DEDENT><DEDENT>encoderAddSet = []<EOL>limitFields = False<EOL>if self._hsObj._maxBranching > <NUM_LIT:0>or self._hsObj._minFieldContribution >= <NUM_LIT:0>:<EOL><INDENT>if self._hsObj._searchType == HsSearchType.temporal orself._hsObj._searchType == HsSearchType.classification:<EOL><INDENT>if sprintIdx >= <NUM_LIT:1>:<EOL><INDENT>limitFields = True<EOL>baseSprintIdx = <NUM_LIT:0><EOL><DEDENT><DEDENT>elif self._hsObj._searchType == HsSearchType.legacyTemporal:<EOL><INDENT>if sprintIdx >= <NUM_LIT:2>:<EOL><INDENT>limitFields = True<EOL>baseSprintIdx = <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(self._hsObj._searchType))<EOL><DEDENT><DEDENT>if limitFields:<EOL><INDENT>pctFieldContributions, absFieldContributions =self.getFieldContributions()<EOL>toRemove = []<EOL>self.logger.debug("<STR_LIT>" %(self._hsObj._minFieldContribution))<EOL>for fieldname in pctFieldContributions:<EOL><INDENT>if pctFieldContributions[fieldname] < self._hsObj._minFieldContribution:<EOL><INDENT>self.logger.debug("<STR_LIT>" % (fieldname))<EOL>toRemove.append(self.getEncoderKeyFromName(fieldname))<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug("<STR_LIT>" % (fieldname))<EOL><DEDENT><DEDENT>swarms = self._state["<STR_LIT>"]<EOL>sprintSwarms = [(swarm, swarms[swarm]["<STR_LIT>"])for swarm in swarms if swarms[swarm]["<STR_LIT>"] == baseSprintIdx]<EOL>sprintSwarms = sorted(sprintSwarms, key=itemgetter(<NUM_LIT:1>))<EOL>if self._hsObj._maxBranching > <NUM_LIT:0>:<EOL><INDENT>sprintSwarms = sprintSwarms[<NUM_LIT:0>:self._hsObj._maxBranching]<EOL><DEDENT>for swarm in sprintSwarms:<EOL><INDENT>swarmEncoders = swarm[<NUM_LIT:0>].split("<STR_LIT:.>")<EOL>for encoder in swarmEncoders:<EOL><INDENT>if not encoder in encoderAddSet:<EOL><INDENT>encoderAddSet.append(encoder)<EOL><DEDENT><DEDENT><DEDENT>encoderAddSet = [encoder for encoder in encoderAddSetif not str(encoder) in toRemove]<EOL><DEDENT>else:<EOL><INDENT>encoderAddSet = self._hsObj._encoderNames<EOL><DEDENT>newSwarmIds = set()<EOL>if (self._hsObj._searchType == HsSearchType.temporalor self._hsObj._searchType == HsSearchType.legacyTemporal)and sprintIdx == <NUM_LIT:2>and (self._hsObj._tryAll3FieldCombinations orself._hsObj._tryAll3FieldCombinationsWTimestamps):<EOL><INDENT>if self._hsObj._tryAll3FieldCombinations:<EOL><INDENT>newEncoders = set(self._hsObj._encoderNames)<EOL>if self._hsObj._predictedFieldEncoder in newEncoders:<EOL><INDENT>newEncoders.remove(self._hsObj._predictedFieldEncoder)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>newEncoders = set(encoderAddSet)<EOL>if self._hsObj._predictedFieldEncoder in newEncoders:<EOL><INDENT>newEncoders.remove(self._hsObj._predictedFieldEncoder)<EOL><DEDENT>for encoder in self._hsObj._encoderNames:<EOL><INDENT>if encoder.endswith('<STR_LIT>') or encoder.endswith('<STR_LIT>')or encoder.endswith('<STR_LIT>'):<EOL><INDENT>newEncoders.add(encoder)<EOL><DEDENT><DEDENT><DEDENT>allCombos = list(itertools.combinations(newEncoders, <NUM_LIT:2>))<EOL>for combo in allCombos:<EOL><INDENT>newSet = list(combo)<EOL>newSet.append(self._hsObj._predictedFieldEncoder)<EOL>newSet.sort()<EOL>newSwarmId = '<STR_LIT:.>'.join(newSet)<EOL>if newSwarmId not in self._state['<STR_LIT>']:<EOL><INDENT>newSwarmIds.add(newSwarmId)<EOL>if (len(self.getActiveSwarms(sprintIdx-<NUM_LIT:1>)) > <NUM_LIT:0>):<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for baseEncoderSet in baseEncoderSets:<EOL><INDENT>for encoder in encoderAddSet:<EOL><INDENT>if encoder not in self._state['<STR_LIT>']and encoder not in baseEncoderSet:<EOL><INDENT>newSet = list(baseEncoderSet)<EOL>newSet.append(encoder)<EOL>newSet.sort()<EOL>newSwarmId = '<STR_LIT:.>'.join(newSet)<EOL>if newSwarmId not in self._state['<STR_LIT>']:<EOL><INDENT>newSwarmIds.add(newSwarmId)<EOL>if (len(self.getActiveSwarms(sprintIdx-<NUM_LIT:1>)) > <NUM_LIT:0>):<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>newSwarmIds = sorted(newSwarmIds)<EOL>if len(newSwarmIds) == <NUM_LIT:0>:<EOL><INDENT>if len(self.getAllSwarms(sprintIdx)) > <NUM_LIT:0>:<EOL><INDENT>return (True, False)<EOL><DEDENT>else:<EOL><INDENT>return (False, True)<EOL><DEDENT><DEDENT>self._dirty = True<EOL>if len(self._state["<STR_LIT>"]) == sprintIdx:<EOL><INDENT>self._state['<STR_LIT>'].append({'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None})<EOL><DEDENT>for swarmId in newSwarmIds:<EOL><INDENT>self._state['<STR_LIT>'][swarmId] = {'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': sprintIdx}<EOL><DEDENT>self._state['<STR_LIT>'] = self.getActiveSwarms()<EOL>success = self.writeStateToDB()<EOL>if success:<EOL><INDENT>return (True, False)<EOL><DEDENT><DEDENT>

If the given sprint exists and is active, return active=True.

        If the sprint does not exist yet, this call will create it (and return
        active=True). If it already exists, but is completing or complete, return
        active=False.

        If sprintIdx is past the end of the possible sprints, return
          active=False, noMoreSprints=True

        IMPORTANT: When speculative particles are enabled, this call has some
        special processing to handle speculative sprints:

          * When creating a new speculative sprint (creating sprint N before
          sprint N-1 has completed), it initially only puts in only ONE swarm into
          the sprint.

          * Every time it is asked if sprint N is active, it also checks to see if
          it is time to add another swarm to the sprint, and adds a new swarm if
          appropriate before returning active=True

          * We decide it is time to add a new swarm to a speculative sprint when ALL
          of the currently active swarms in the sprint have all the workers they
          need (number of running (not mature) particles is _minParticlesPerSwarm).
          This means that we have capacity to run additional particles in a new
          swarm.

        It is expected that the sprints will be checked IN ORDER from 0 on up. (It
        is an error not to) The caller should always try to allocate from the first
        active sprint it finds. If it can't, then it can call this again to
        find/create the next active sprint.

        Parameters:
        ---------------------------------------------------------------------
        retval:   (active, noMoreSprints)
                    active: True if the given sprint is active
                    noMoreSprints: True if there are no more sprints possible


self._baseLogger.debug(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'DEBUG'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)


self._baseLogger.info(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'INFO'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.info("Houston, we have a %s", "interesting problem", exc_info=1)


def warning(self, msg, *args, **kwargs):

self._baseLogger.warning(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'WARNING'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)


self._baseLogger.error(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'ERROR'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.error("Houston, we have a %s", "major problem", exc_info=1)


def critical(self, msg, *args, **kwargs):

self._baseLogger.critical(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'CRITICAL'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.critical("Houston, we have a %s", "major disaster", exc_info=1)


def log(self, level, msg, *args, **kwargs):

self._baseLogger.log(self, level, self.getExtendedMsg(msg), *args,<EOL>**kwargs)<EOL>

Log 'msg % args' with the integer severity 'level'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.log(level, "We have a %s", "mysterious problem", exc_info=1)


activities = [None] * len(ModelTerminator._MILESTONES)<EOL>for index, (iteration, _) in enumerate(ModelTerminator._MILESTONES):<EOL><INDENT>cb = functools.partial(terminationFunc, index=index)<EOL>activities[index] = PeriodicActivityRequest(repeating =False,<EOL>period = iteration,<EOL>cb=cb)<EOL><DEDENT>

Returns the periodic checks to see if the model should
        continue running.

        Parameters:
        -----------------------------------------------------------------------
        terminationFunc:  The function that will be called in the model main loop
                          as a wrapper around this function. Must have a parameter
                          called 'index'

        Returns:          A list of PeriodicActivityRequest objects.


def recordDataPoint(self, swarmId, generation, errScore):

terminatedSwarms = []<EOL>if swarmId in self.swarmScores:<EOL><INDENT>entry = self.swarmScores[swarmId]<EOL>assert(len(entry) == generation)<EOL>entry.append(errScore)<EOL>entry = self.swarmBests[swarmId]<EOL>entry.append(min(errScore, entry[-<NUM_LIT:1>]))<EOL>assert(len(self.swarmBests[swarmId]) == len(self.swarmScores[swarmId]))<EOL><DEDENT>else:<EOL><INDENT>assert (generation == <NUM_LIT:0>)<EOL>self.swarmScores[swarmId] = [errScore]<EOL>self.swarmBests[swarmId] = [errScore]<EOL><DEDENT>if generation + <NUM_LIT:1> < self.MATURITY_WINDOW:<EOL><INDENT>return terminatedSwarms<EOL><DEDENT>if self.MAX_GENERATIONS is not None and generation > self.MAX_GENERATIONS:<EOL><INDENT>self._logger.info(<EOL>'<STR_LIT>' %<EOL>(swarmId, self.MAX_GENERATIONS))<EOL>terminatedSwarms.append(swarmId)<EOL><DEDENT>if self._isTerminationEnabled:<EOL><INDENT>terminatedSwarms.extend(self._getTerminatedSwarms(generation))<EOL><DEDENT>cumulativeBestScores = self.swarmBests[swarmId]<EOL>if cumulativeBestScores[-<NUM_LIT:1>] == cumulativeBestScores[-self.MATURITY_WINDOW]:<EOL><INDENT>self._logger.info('<STR_LIT>'<EOL>'<STR_LIT>'% (swarmId, self.MATURITY_WINDOW))<EOL>terminatedSwarms.append(swarmId)<EOL><DEDENT>self.terminatedSwarms = self.terminatedSwarms.union(terminatedSwarms)<EOL>return terminatedSwarms<EOL>

Record the best score for a swarm's generation index (x)
        Returns list of swarmIds to terminate.


def getLabel(cls, val):<EOL><INDENT>"""<STR_LIT>"""<EOL>return cls.__labels[val]<EOL><DEDENT>def validate(cls, val):<EOL><INDENT>"""<STR_LIT>"""<EOL>return val in cls.__values<EOL><DEDENT>def getValues(cls):<EOL><INDENT>"""<STR_LIT>"""<EOL>return list(cls.__values)<EOL><DEDENT>def getLabels(cls):<EOL><INDENT>"""<STR_LIT>"""<EOL>return list(cls.__labels.values())<EOL><DEDENT>def getValue(cls, label):<EOL><INDENT>"""<STR_LIT>"""<EOL>return cls.__labels[label]<EOL><DEDENT>for arg in list(args)+list(kwargs.keys()):<EOL><INDENT>if type(arg) is not str:<EOL><INDENT>raise TypeError("<STR_LIT>".format(arg))<EOL><DEDENT>if not __isidentifier(arg):<EOL><INDENT>raise ValueError("<STR_LIT>""<STR_LIT>".format(arg))<EOL><DEDENT><DEDENT>kwargs.update(list(zip(args, args)))<EOL>newType = type("<STR_LIT>", (object,), kwargs)<EOL>newType.__labels = dict( (v,k) for k,v in kwargs.items())<EOL>newType.__values = set(newType.__labels.keys())<EOL>newType.getLabel = functools.partial(getLabel, newType)<EOL>newType.validate = functools.partial(validate, newType)<EOL>newType.getValues = functools.partial(getValues, newType)<EOL>newType.getLabels = functools.partial(getLabels, newType)<EOL>newType.getValue = functools.partial(getValue, newType)<EOL>return newType<EOL>

Utility function for creating enumerations in python

Example Usage:
  >> Color = Enum("Red", "Green", "Blue", "Magenta")
  >> print Color.Red
  >> 0
  >> print Color.Green
  >> 1
  >> print Color.Blue
  >> 2
  >> print Color.Magenta
  >> 3
  >> Color.Violet
  >> 'violet'
  >> Color.getLabel(Color.Red)
  >> 'Red'
  >> Color.getLabel(2)
  >> 'Blue'


assert os.path.isabs(absDirPath)<EOL>try:<EOL><INDENT>os.makedirs(absDirPath)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != os.errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return absDirPath<EOL>

Makes directory for the given directory path with default permissions.
    If the directory already exists, it is treated as success.

    absDirPath:   absolute path of the directory to create.

    Returns:      absDirPath arg

    Exceptions:         OSError if directory creation fails


if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>envValue = os.environ.get("<STR_LIT>" % (cls.envPropPrefix,<EOL>prop.replace('<STR_LIT:.>', '<STR_LIT:_>')), None)<EOL>if envValue is not None:<EOL><INDENT>return envValue<EOL><DEDENT>return cls._properties[prop]<EOL>

Retrieve the requested property as a string. If property does not exist,
        then KeyError will be raised.

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        retval:      property value as a string


value = cls.getInt(prop)<EOL>if value not in (<NUM_LIT:0>, <NUM_LIT:1>):<EOL><INDENT>raise ValueError("<STR_LIT>" % (<EOL>value, prop))<EOL><DEDENT>return bool(value)<EOL>

Retrieve the requested property and return it as a bool. If property
        does not exist, then KeyError will be raised. If the property value is
        neither 0 nor 1, then ValueError will be raised

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        retval:      property value as bool


return int(cls.getString(prop))<EOL>

Retrieve the requested property and return it as an int. If property
        does not exist, then KeyError will be raised.

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        retval:      property value as int


return float(cls.getString(prop))<EOL>

Retrieve the requested property and return it as a float. If property
        does not exist, then KeyError will be raised.

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        retval:      property value as float


@classmethod<EOL><INDENT>def get(cls, prop, default=None):<DEDENT>

try:<EOL><INDENT>return cls.getString(prop)<EOL><DEDENT>except KeyError:<EOL><INDENT>return default<EOL><DEDENT>

Get the value of the given configuration property as string. This
        returns a string which is the property value, or the value of "default" arg
        if the property is not found. Use Configuration.getString() instead.

        NOTE: it's atypical for our configuration properties to be missing - a
         missing configuration property is usually a very serious error. Because
         of this, it's preferable to use one of the getString, getInt, getFloat,
         etc. variants instead of get(). Those variants will raise KeyError when
         an expected property is missing.

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        default:     default value to return if property does not exist
        retval:      property value (as a string), or default if the property does
                      not exist.


@classmethod<EOL><INDENT>def set(cls, prop, value):<DEDENT>

if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>cls._properties[prop] = str(value)<EOL>

Set the value of the given configuration property.

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        value:       value to set


if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>result = dict(cls._properties)<EOL>keys = list(os.environ.keys())<EOL>replaceKeys = [x for x in keys if x.startswith(cls.envPropPrefix)]<EOL>for envKey in replaceKeys:<EOL><INDENT>key = envKey[len(cls.envPropPrefix):]<EOL>key = key.replace('<STR_LIT:_>', '<STR_LIT:.>')<EOL>result[key] = os.environ[envKey]<EOL><DEDENT>return result<EOL>

Return a dict containing all of the configuration properties

        Parameters:
        ----------------------------------------------------------------
        retval:      dict containing all configuration properties.


@classmethod<EOL><INDENT>def readConfigFile(cls, filename, path=None):<DEDENT>

properties = cls._readConfigFile(filename, path)<EOL>if cls._properties is None:<EOL><INDENT>cls._properties = dict()<EOL><DEDENT>for name in properties:<EOL><INDENT>if '<STR_LIT:value>' in properties[name]:<EOL><INDENT>cls._properties[name] = properties[name]['<STR_LIT:value>']<EOL><DEDENT><DEDENT>

Parse the given XML file and store all properties it describes.

        Parameters:
        ----------------------------------------------------------------
        filename:  name of XML file to parse (no path)
        path:      path of the XML file. If None, then use the standard
                      configuration search path.


@classmethod<EOL><INDENT>def _readConfigFile(cls, filename, path=None):<DEDENT>

outputProperties = dict()<EOL>if path is None:<EOL><INDENT>filePath = cls.findConfigFile(filename)<EOL><DEDENT>else:<EOL><INDENT>filePath = os.path.join(path, filename)<EOL><DEDENT>try:<EOL><INDENT>if filePath is not None:<EOL><INDENT>try:<EOL><INDENT>_getLoggerBase().debug("<STR_LIT>", filePath)<EOL>with open(filePath, '<STR_LIT:r>') as inp:<EOL><INDENT>contents = inp.read()<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % filePath)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>contents = resource_string("<STR_LIT>", filename)<EOL><DEDENT>except Exception as resourceException:<EOL><INDENT>if filename in [USER_CONFIG, CUSTOM_CONFIG]:<EOL><INDENT>contents = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>raise resourceException<EOL><DEDENT><DEDENT><DEDENT>elements = ElementTree.XML(contents)<EOL>if elements.tag != '<STR_LIT>':<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (elements.tag))<EOL><DEDENT>propertyElements = elements.findall('<STR_LIT>')<EOL>for propertyItem in propertyElements:<EOL><INDENT>propInfo = dict()<EOL>propertyAttributes = list(propertyItem)<EOL>for propertyAttribute in propertyAttributes:<EOL><INDENT>propInfo[propertyAttribute.tag] = propertyAttribute.text<EOL><DEDENT>name = propInfo.get('<STR_LIT:name>', None)<EOL>if '<STR_LIT:value>' in propInfo and propInfo['<STR_LIT:value>'] is None:<EOL><INDENT>value = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>value = propInfo.get('<STR_LIT:value>', None)<EOL>if value is None:<EOL><INDENT>if '<STR_LIT>' in propInfo:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (str(propInfo)))<EOL><DEDENT><DEDENT><DEDENT>restOfValue = value<EOL>value = '<STR_LIT>'<EOL>while True:<EOL><INDENT>pos = restOfValue.find('<STR_LIT>')<EOL>if pos == -<NUM_LIT:1>:<EOL><INDENT>value += restOfValue<EOL>break<EOL><DEDENT>value += restOfValue[<NUM_LIT:0>:pos]<EOL>varTailPos = restOfValue.find('<STR_LIT:}>', pos)<EOL>if varTailPos == -<NUM_LIT:1>:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (restOfValue))<EOL><DEDENT>varname = restOfValue[pos + <NUM_LIT:6>:varTailPos]<EOL>if varname not in os.environ:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>varname))<EOL><DEDENT>envVarValue = os.environ[varname]<EOL>value += envVarValue<EOL>restOfValue = restOfValue[varTailPos + <NUM_LIT:1>:]<EOL><DEDENT>if name is None:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (str(propInfo)))<EOL><DEDENT>propInfo['<STR_LIT:value>'] = value<EOL>outputProperties[name] = propInfo<EOL><DEDENT>return outputProperties<EOL><DEDENT>except Exception:<EOL><INDENT>_getLoggerBase().exception("<STR_LIT>",<EOL>filePath)<EOL>raise<EOL><DEDENT>

Parse the given XML file and return a dict describing the file.

        Parameters:
        ----------------------------------------------------------------
        filename:  name of XML file to parse (no path)
        path:      path of the XML file. If None, then use the standard
                      configuration search path.
        retval:    returns a dict with each property as a key and a dict of all
                   the property's attributes as value


cls._properties = None<EOL>cls._configPaths = None<EOL>


paths = cls.getConfigPaths()<EOL>for p in paths:<EOL><INDENT>testPath = os.path.join(p, filename)<EOL>if os.path.isfile(testPath):<EOL><INDENT>return os.path.join(p, filename)<EOL><DEDENT><DEDENT>

Search the configuration path (specified via the NTA_CONF_PATH
        environment variable) for the given filename. If found, return the complete
        path to the file.

        Parameters:
        ----------------------------------------------------------------
        filename:  name of file to locate


configPaths = []<EOL>if cls._configPaths is not None:<EOL><INDENT>return cls._configPaths<EOL><DEDENT>else:<EOL><INDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>configVar = os.environ['<STR_LIT>']<EOL>configPaths = configVar.split(os.pathsep)<EOL><DEDENT>return configPaths<EOL><DEDENT>

Return the list of paths to search for configuration files.

        Parameters:
        ----------------------------------------------------------------
        retval:    list of paths.


cls._configPaths = list(paths)<EOL>

Modify the paths we use to search for configuration files.

        Parameters:
        ----------------------------------------------------------------
        paths:   list of paths to search for config files.


<EOL>cls.readConfigFile(DEFAULT_CONFIG)<EOL>cls.readConfigFile(USER_CONFIG)<EOL>

Read in all standard configuration files


return _CustomConfigurationFileWrapper.getCustomDict()<EOL>

Return a dict containing all custom configuration properties

        Parameters:
        ----------------------------------------------------------------
        retval:      dict containing all custom configuration properties.


@classmethod<EOL><INDENT>def setCustomProperty(cls, propertyName, value):<DEDENT>

cls.setCustomProperties({propertyName: value})<EOL>

Set a single custom setting and persist it to the custom
        configuration store.

        Parameters:
        ----------------------------------------------------------------
        propertyName: string containing the name of the property to get
        value: value to set the property to


_getLogger().info("<STR_LIT>",<EOL>properties, traceback.format_stack())<EOL>_CustomConfigurationFileWrapper.edit(properties)<EOL>for propertyName, value in properties.items():<EOL><INDENT>cls.set(propertyName, value)<EOL><DEDENT>

Set multiple custom properties and persist them to the custom
        configuration store.

        Parameters:
        ----------------------------------------------------------------
        properties: a dict of property name/value pairs to set


<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=False)<EOL>

Clear all configuration properties from in-memory cache, but do NOT
        alter the custom configuration file. Used in unit-testing.


_getLogger().info("<STR_LIT>"<EOL>"<STR_LIT>", traceback.format_stack())<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=True)<EOL>

Clear all custom configuration settings and delete the persistent
        custom configuration store.


cls.readConfigFile(_CustomConfigurationFileWrapper.customFileName)<EOL>

Loads custom configuration settings from their persistent storage.
        DO NOT CALL THIS: It's typically not necessary to call this method
        directly - see NOTE below.

        NOTE: this method exists *solely* for the benefit of prepare_conf.py, which
        needs to load configuration files selectively.


super(Configuration, cls)._readStdConfigFiles()<EOL>cls.loadCustomConfig()<EOL>

Intercept the _readStdConfigFiles call from our base config class to
        read in base and custom configuration settings.


if persistent:<EOL><INDENT>try:<EOL><INDENT>os.unlink(cls.getPath())<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != errno.ENOENT:<EOL><INDENT>_getLogger().exception("<STR_LIT>""<STR_LIT>", e.errno,<EOL>cls.getPath())<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>cls._path = None<EOL>

If persistent is True, delete the temporary file

        Parameters:
        ----------------------------------------------------------------
        persistent: if True, custom configuration file is deleted


if not os.path.exists(cls.getPath()):<EOL><INDENT>return dict()<EOL><DEDENT>properties = Configuration._readConfigFile(os.path.basename(<EOL>cls.getPath()), os.path.dirname(cls.getPath()))<EOL>values = dict()<EOL>for propName in properties:<EOL><INDENT>if '<STR_LIT:value>' in properties[propName]:<EOL><INDENT>values[propName] = properties[propName]['<STR_LIT:value>']<EOL><DEDENT><DEDENT>return values<EOL>

Returns a dict of all temporary values in custom configuration file


copyOfProperties = copy(properties)<EOL>configFilePath = cls.getPath()<EOL>try:<EOL><INDENT>with open(configFilePath, '<STR_LIT:r>') as fp:<EOL><INDENT>contents = fp.read()<EOL><DEDENT><DEDENT>except IOError as e:<EOL><INDENT>if e.errno != errno.ENOENT:<EOL><INDENT>_getLogger().exception("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>e.errno, configFilePath, properties)<EOL>raise<EOL><DEDENT>contents = '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>elements = ElementTree.XML(contents)<EOL>ElementTree.tostring(elements)<EOL><DEDENT>except Exception as e:<EOL><INDENT>msg = "<STR_LIT>""<STR_LIT>" %(configFilePath, contents, type(e), e)<EOL>_getLogger().exception(msg)<EOL>raise RuntimeError(msg).with_traceback(sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>if elements.tag != '<STR_LIT>':<EOL><INDENT>e = "<STR_LIT>" %(elements.tag)<EOL>_getLogger().error(e)<EOL>raise RuntimeError(e)<EOL><DEDENT>for propertyItem in elements.findall('<STR_LIT>'):<EOL><INDENT>propInfo = dict((attr.tag, attr.text) for attr in propertyItem)<EOL>name = propInfo['<STR_LIT:name>']<EOL>if name in copyOfProperties:<EOL><INDENT>foundValues = propertyItem.findall('<STR_LIT>')<EOL>if len(foundValues) > <NUM_LIT:0>:<EOL><INDENT>foundValues[<NUM_LIT:0>].text = str(copyOfProperties.pop(name))<EOL>if not copyOfProperties:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>e = "<STR_LIT>" % (name,)<EOL>_getLogger().error(e)<EOL>raise RuntimeError(e)<EOL><DEDENT><DEDENT><DEDENT>for propertyName, value in copyOfProperties.items():<EOL><INDENT>newProp = ElementTree.Element('<STR_LIT>')<EOL>nameTag = ElementTree.Element('<STR_LIT:name>')<EOL>nameTag.text = propertyName<EOL>newProp.append(nameTag)<EOL>valueTag = ElementTree.Element('<STR_LIT:value>')<EOL>valueTag.text = str(value)<EOL>newProp.append(valueTag)<EOL>elements.append(newProp)<EOL><DEDENT>try:<EOL><INDENT>makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))<EOL>with open(configFilePath, '<STR_LIT:w>') as fp:<EOL><INDENT>fp.write(ElementTree.tostring(elements))<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>_getLogger().exception("<STR_LIT>"<EOL>"<STR_LIT>", properties,<EOL>configFilePath)<EOL>raise<EOL><DEDENT>

Edits the XML configuration file with the parameters specified by
        properties

        Parameters:
        ----------------------------------------------------------------
        properties: dict of settings to be applied to the custom configuration store
                     (key is property name, value is value)


cls._path = os.path.join(os.environ['<STR_LIT>'],<EOL>cls.customFileName)<EOL>

Sets the path of the custom configuration file


if cls._path is None:<EOL><INDENT>cls._setPath()<EOL><DEDENT>return cls._path<EOL>

Get the path of the custom configuration file


super(OPFDummyModelRunner, self).__init__(modelID=modelID,<EOL>jobID=jobID,<EOL>predictedField=predictedField,<EOL>experimentDir=None,<EOL>reportKeyPatterns=reportKeyPatterns,<EOL>optimizeKeyPattern=optimizeKeyPattern,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=None)<EOL>self._predictionCacheMaxRecords = predictionCacheMaxRecords<EOL>self._streamDef = copy.deepcopy(self._DUMMY_STREAMDEF)<EOL>self._params = copy.deepcopy(self._DEFAULT_PARAMS)<EOL>if '<STR_LIT>' in paramsand '<STR_LIT>' in params['<STR_LIT>']:<EOL><INDENT>self.modelIndex=params['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>self.modelIndex = OPFDummyModelRunner.modelIndex<EOL>OPFDummyModelRunner.modelIndex += <NUM_LIT:1><EOL><DEDENT>self._loadDummyModelParameters(params)<EOL>self._logger.debug("<STR_LIT>", self._params)<EOL>self._busyWaitTime = self._params['<STR_LIT>']<EOL>self._iterations = self._params['<STR_LIT>']<EOL>self._doFinalize = self._params['<STR_LIT>']<EOL>self._delay = self._params['<STR_LIT>']<EOL>self._sleepModelRange = self._params['<STR_LIT>']<EOL>self._makeCheckpoint = self._params['<STR_LIT>']<EOL>self._finalDelay = self._params['<STR_LIT>']<EOL>self._exitAfter = self._params['<STR_LIT>']<EOL>self.randomizeWait = self._params['<STR_LIT>']<EOL>if self._busyWaitTime is not None:<EOL><INDENT>self.__computeWaitTime()<EOL><DEDENT>if self._params['<STR_LIT>'] is not Noneand self._params['<STR_LIT>'] is not None:<EOL><INDENT>raise RuntimeError("<STR_LIT>""<STR_LIT>")<EOL><DEDENT>self.metrics = None<EOL>self.metricValue = None<EOL>if self._params['<STR_LIT>'] is not None:<EOL><INDENT>self.metrics = eval(self._params['<STR_LIT>'])<EOL><DEDENT>elif self._params['<STR_LIT>'] is not None:<EOL><INDENT>self.metricValue = float(self._params['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>self.metrics = OPFDummyModelRunner.metrics[<NUM_LIT:0>]<EOL><DEDENT>if self._params['<STR_LIT>'] is not None:<EOL><INDENT>self._model = self.__createModel(self._params['<STR_LIT>'])<EOL>self.__fieldInfo = self._model.getFieldInfo()<EOL><DEDENT>self._sysExitModelRange = self._params['<STR_LIT>']<EOL>if self._sysExitModelRange is not None:<EOL><INDENT>self._sysExitModelRange = [int(x) for x in self._sysExitModelRange.split('<STR_LIT:U+002C>')]<EOL><DEDENT>self._delayModelRange = self._params['<STR_LIT>']<EOL>if self._delayModelRange is not None:<EOL><INDENT>self._delayModelRange = [int(x) for x in self._delayModelRange.split('<STR_LIT:U+002C>')]<EOL><DEDENT>self._errModelRange = self._params['<STR_LIT>']<EOL>if self._errModelRange is not None:<EOL><INDENT>self._errModelRange = [int(x) for x in self._errModelRange.split('<STR_LIT:U+002C>')]<EOL><DEDENT>self._computModelDelay()<EOL>self._jobFailErr = self._params['<STR_LIT>']<EOL>self._logger.debug("<STR_LIT>", self._modelID, self._params)<EOL>

Parameters:
-------------------------------------------------------------------------
modelID:    ID of this model in the models table

jobID:

params:     a dictionary of parameters for this dummy model. The
            possible keys are:

              delay:          OPTIONAL-This specifies the amount of time
                              (in seconds) that the experiment should wait
                              before STARTING to process records. This is
                              useful for simulating workers that start/end
                              at different times

              finalDelay:     OPTIONAL-This specifies the amount of time
                              (in seconds) that the experiment should wait
                              before it conducts its finalization operations.
                              These operations include checking if the model
                              is the best model, and writing out checkpoints.

              waitTime:       OPTIONAL-The amount of time (in seconds)
                              to wait in a busy loop to simulate
                              computation time on EACH ITERATION

              randomizeWait:  OPTIONAL-([0.0-1.0] ). Default:None
                              If set to a value, the above specified
                              wait time will be randomly be dithered by
                              +/- <randomizeWait>% of the specfied value.
                              For example, if randomizeWait=0.2, the wait
                              time will be dithered by +/- 20% of its value.

              iterations:     OPTIONAL-How many iterations to run the model
                              for. -1 means run forever (default=1)

              metricFunctions: OPTIONAL-A list of single argument functions
                               serialized as strings, which return the metric
                               value given the record number.

                               Mutually exclusive with metricValue

              metricValue:    OPTIONAL-A single value to use for the metric
                              value (used to debug hypersearch).

                              Mutually exclusive with metricFunctions

              finalize:       OPTIONAL-(True/False). Default:True
                              When False, this will prevent the model from
                              recording it's metrics and performing other
                              functions that it usually performs after the
                              model has finished running

              permutationParams: A dict containing the instances of all the
                                  variables being permuted over

              experimentDirectory: REQUIRED-An absolute path to a directory
                                   with a valid description.py file.

                                   NOTE: This does not actually affect the
                                   running of the model or the metrics
                                   produced. It is required to create certain
                                   objects (such as the output stream)

              makeCheckpoint:     True to actually write a checkpoint out to
                                  disk (default: False)

              sysExitModelRange: A string containing two integers 'firstIdx,
                              endIdx'. When present, if we are running the
                              firstIdx'th model up to but not including the
                              endIdx'th model, then do a sys.exit() while
                              running the model. This causes the worker to
                              exit, simulating an orphaned model.

              delayModelRange: A string containing two integers 'firstIdx,
                              endIdx'. When present, if we are running the
                              firstIdx'th model up to but not including the
                              endIdx'th model, then do a delay of 10 sec.
                              while running the model. This causes the
                              worker to run slower and for some other worker
                              to think the model should be orphaned.

              exitAfter:      The number of iterations after which the model
                              should perform a sys exit. This is an
                              alternative way of creating an orphaned model
                              that use's the dummmy model's modelIndex
                              instead of the modelID

              errModelRange: A string containing two integers 'firstIdx,
                              endIdx'. When present, if we are running the
                              firstIdx'th model up to but not including the
                              endIdx'th model, then raise an exception while
                              running the model. This causes the model to
                              fail with a CMPL_REASON_ERROR reason

              sleepModelRange: A string containing 3 integers 'firstIdx,
                              endIdx: delay'. When present, if we are running
                              the firstIdx'th model up to but not including
                              the endIdx'th model, then sleep for delay
                              seconds at the beginning of the run.

              jobFailErr: If true, model will raise a JobFailException
                          which should cause the job to be marked as
                          failed and immediately cancel all other workers.

predictedField:     Name of the input field for which this model is being
                    optimized

reportKeyPatterns:  list of items from the results dict to include in
                    the report. These can be regular expressions.
optimizeKeyPattern: Which report item, if any, we will be optimizing for.
                    This can also be a regular expression, but is an error
                    if it matches more than one key from the experiment's
                    results.

jobsDAO:            Jobs data access object - the interface to the
                    jobs database which has the model's table.

modelCheckpointGUID:
                    A persistent, globally-unique identifier for
                    constructing the model checkpoint key

logLevel:           override logging level to this value, if not None
predictionCacheMaxRecords:
                    Maximum number of records for the prediction output cache.
                    Pass None for the default value.


for key, value in params.iteritems():<EOL><INDENT>if type(value) == list:<EOL><INDENT>index = self.modelIndex % len(params[key])<EOL>self._params[key] = params[key][index]<EOL><DEDENT>else:<EOL><INDENT>self._params[key] = params[key]<EOL><DEDENT><DEDENT>

Loads all the parameters for this dummy model. For any paramters
        specified as lists, read the appropriate value for this model using the model
        index


<EOL>if self._params['<STR_LIT>'] is not Noneand self._params['<STR_LIT>'] is not None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if self._sleepModelRange is not None:<EOL><INDENT>range, delay = self._sleepModelRange.split('<STR_LIT::>')<EOL>delay = float(delay)<EOL>range = map(int, range.split('<STR_LIT:U+002C>'))<EOL>modelIDs = self._jobsDAO.jobGetModelIDs(self._jobID)<EOL>modelIDs.sort()<EOL>range[<NUM_LIT:1>] = min(range[<NUM_LIT:1>], len(modelIDs))<EOL>if self._modelID in modelIDs[range[<NUM_LIT:0>]:range[<NUM_LIT:1>]]:<EOL><INDENT>self._delay = delay<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._delay = self._params['<STR_LIT>']<EOL><DEDENT>

Computes the amount of time (if any) to delay the run of this model.
        This can be determined by two mutually exclusive parameters:
        delay and sleepModelRange.

        'delay' specifies the number of seconds a model should be delayed. If a list
        is specified, the appropriate amount of delay is determined by using the
        model's modelIndex property.

        However, this doesn't work when testing orphaned models, because the
        modelIndex will be the same for every recovery attempt. Therefore, every
        recovery attempt will also be delayed and potentially orphaned.

        'sleepModelRange' doesn't use the modelIndex property for a model, but rather
        sees which order the model is in the database, and uses that to determine
        whether or not a model should be delayed.


metric = None<EOL>if self.metrics is not None:<EOL><INDENT>metric = self.metrics(self._currentRecordIndex+<NUM_LIT:1>)<EOL><DEDENT>elif self.metricValue is not None:<EOL><INDENT>metric = self.metricValue<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>return {self._optimizeKeyPattern:metric}<EOL>

Protected function that can be overridden by subclasses. Its main purpose
        is to allow the the OPFDummyModelRunner to override this with deterministic
        values

        Returns: All the metrics being computed for this model


self._logger.debug("<STR_LIT>" % (self._modelID))<EOL>periodic = self._initPeriodicActivities()<EOL>self._optimizedMetricLabel = self._optimizeKeyPattern<EOL>self._reportMetricLabels = [self._optimizeKeyPattern]<EOL>if self._iterations >= <NUM_LIT:0>:<EOL><INDENT>iterTracker = iter(xrange(self._iterations))<EOL><DEDENT>else:<EOL><INDENT>iterTracker = iter(itertools.count())<EOL><DEDENT>doSysExit = False<EOL>if self._sysExitModelRange is not None:<EOL><INDENT>modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)<EOL>modelIDs = [x[<NUM_LIT:0>] for x in modelAndCounters]<EOL>modelIDs.sort()<EOL>(beg,end) = self._sysExitModelRange<EOL>if self._modelID in modelIDs[int(beg):int(end)]:<EOL><INDENT>doSysExit = True<EOL><DEDENT><DEDENT>if self._delayModelRange is not None:<EOL><INDENT>modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)<EOL>modelIDs = [x[<NUM_LIT:0>] for x in modelAndCounters]<EOL>modelIDs.sort()<EOL>(beg,end) = self._delayModelRange<EOL>if self._modelID in modelIDs[int(beg):int(end)]:<EOL><INDENT>time.sleep(<NUM_LIT:10>)<EOL><DEDENT><DEDENT>if self._errModelRange is not None:<EOL><INDENT>modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)<EOL>modelIDs = [x[<NUM_LIT:0>] for x in modelAndCounters]<EOL>modelIDs.sort()<EOL>(beg,end) = self._errModelRange<EOL>if self._modelID in modelIDs[int(beg):int(end)]:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>if self._delay is not None:<EOL><INDENT>time.sleep(self._delay)<EOL><DEDENT>self._currentRecordIndex = <NUM_LIT:0><EOL>while True:<EOL><INDENT>if self._isKilled:<EOL><INDENT>break<EOL><DEDENT>if self._isCanceled:<EOL><INDENT>break<EOL><DEDENT>if self._isMature:<EOL><INDENT>if not self._isBestModel:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED<EOL>break<EOL><DEDENT>else:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_EOF<EOL><DEDENT><DEDENT>try:<EOL><INDENT>self._currentRecordIndex = next(iterTracker)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>self._writePrediction(ModelResult(None, None, None, None))<EOL>periodic.tick()<EOL>if self.__shouldSysExit(self._currentRecordIndex):<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if self._busyWaitTime is not None:<EOL><INDENT>time.sleep(self._busyWaitTime)<EOL>self.__computeWaitTime()<EOL><DEDENT>if doSysExit:<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if self._jobFailErr:<EOL><INDENT>raise utils.JobFailException("<STR_LIT>",<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>if self._doFinalize:<EOL><INDENT>if not self._makeCheckpoint:<EOL><INDENT>self._model = None<EOL><DEDENT>if self._finalDelay is not None:<EOL><INDENT>time.sleep(self._finalDelay)<EOL><DEDENT>self._finalize()<EOL><DEDENT>self._logger.info("<STR_LIT>"% (self._modelID))<EOL>return (self._cmpReason, None)<EOL>

Runs the given OPF task against the given Model instance


class DummyLogger:<EOL><INDENT>def writeRecord(self, record): pass<EOL>def writeRecords(self, records, progressCB): pass<EOL>def close(self): pass<EOL><DEDENT>self._predictionLogger = DummyLogger()<EOL>

Creates the model's PredictionLogger object, which is an interface to write
model results to a permanent storage location


if self._exitAfter is Noneor iteration < self._exitAfter:<EOL><INDENT>return False<EOL><DEDENT>results = self._jobsDAO.modelsGetFieldsForJob(self._jobID, ['<STR_LIT>'])<EOL>modelIDs = [e[<NUM_LIT:0>] for e in results]<EOL>modelNums = [json.loads(e[<NUM_LIT:1>][<NUM_LIT:0>])['<STR_LIT>']['<STR_LIT>'] for e in results]<EOL>sameModelNumbers = filter(lambda x: x[<NUM_LIT:1>] == self.modelIndex,<EOL>zip(modelIDs, modelNums))<EOL>firstModelID = min(zip(*sameModelNumbers)[<NUM_LIT:0>])<EOL>return firstModelID == self._modelID<EOL>

Checks to see if the model should exit based on the exitAfter dummy
parameter


<EOL>self._MIN_RECORDS_TO_BE_BEST = int(Configuration.get('<STR_LIT>'))<EOL>self._MATURITY_MAX_CHANGE = float(Configuration.get('<STR_LIT>'))<EOL>self._MATURITY_NUM_POINTS = int(Configuration.get('<STR_LIT>'))<EOL>self._modelID = modelID<EOL>self._jobID = jobID<EOL>self._predictedField = predictedField<EOL>self._experimentDir = experimentDir<EOL>self._reportKeyPatterns = reportKeyPatterns<EOL>self._optimizeKeyPattern = optimizeKeyPattern<EOL>self._jobsDAO = jobsDAO<EOL>self._modelCheckpointGUID = modelCheckpointGUID<EOL>self._predictionCacheMaxRecords = predictionCacheMaxRecords<EOL>self._isMaturityEnabled = bool(int(Configuration.get('<STR_LIT>')))<EOL>self._logger = logging.getLogger("<STR_LIT:.>".join( ['<STR_LIT>',<EOL>self.__class__.__module__, self.__class__.__name__]))<EOL>self._optimizedMetricLabel = None<EOL>self._reportMetricLabels = []<EOL>self._cmpReason = ClientJobsDAO.CMPL_REASON_EOF<EOL>if logLevel is not None:<EOL><INDENT>self._logger.setLevel(logLevel)<EOL><DEDENT>self.__metricMgr = None<EOL>self.__task = None<EOL>self._periodic = None<EOL>self._streamDef = None<EOL>self._model = None<EOL>self._inputSource = None<EOL>self._currentRecordIndex = None<EOL>self._predictionLogger = None<EOL>self.__predictionCache = deque()<EOL>self._isBestModel = False<EOL>self._isBestModelStored = False<EOL>self._isCanceled = False<EOL>self._isKilled = False<EOL>self._isMature = False<EOL>self._isInterrupted = threading.Event()<EOL>self._metricRegression = regression.AveragePctChange(windowSize=self._MATURITY_NUM_POINTS)<EOL>self.__loggedMetricPatterns = []<EOL>

Parameters:
-------------------------------------------------------------------------
modelID:            ID for this model in the models table

jobID:              ID for this hypersearch job in the jobs table
predictedField:     Name of the input field for which this model is being
                    optimized
experimentDir:      Directory path containing the experiment's
                    description.py script
reportKeyPatterns:  list of items from the results dict to include in
                    the report. These can be regular expressions.
optimizeKeyPattern: Which report item, if any, we will be optimizing for.
                    This can also be a regular expression, but is an error
                    if it matches more than one key from the experiment's
                    results.
jobsDAO:            Jobs data access object - the interface to the
                    jobs database which has the model's table.
modelCheckpointGUID:
                    A persistent, globally-unique identifier for
                    constructing the model checkpoint key. If None, then
                    don't bother creating a model checkpoint.
logLevel:           override logging level to this value, if not None
predictionCacheMaxRecords:
                    Maximum number of records for the prediction output cache.
                    Pass None for default value.


<EOL>descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(<EOL>self._experimentDir)<EOL>expIface = helpers.getExperimentDescriptionInterfaceFromModule(<EOL>descriptionPyModule)<EOL>expIface.normalizeStreamSources()<EOL>modelDescription = expIface.getModelDescription()<EOL>self._modelControl = expIface.getModelControl()<EOL>streamDef = self._modelControl['<STR_LIT>']<EOL>from nupic.data.stream_reader import StreamReader<EOL>readTimeout = <NUM_LIT:0><EOL>self._inputSource = StreamReader(streamDef, isBlocking=False,<EOL>maxTimeout=readTimeout)<EOL>fieldStats = self._getFieldStats()<EOL>self._model = ModelFactory.create(modelDescription)<EOL>self._model.setFieldStatistics(fieldStats)<EOL>self._model.enableLearning()<EOL>self._model.enableInference(self._modelControl.get("<STR_LIT>", None))<EOL>self.__metricMgr = MetricsManager(self._modelControl.get('<STR_LIT>',None),<EOL>self._model.getFieldInfo(),<EOL>self._model.getInferenceType())<EOL>self.__loggedMetricPatterns = self._modelControl.get("<STR_LIT>", [])<EOL>self._optimizedMetricLabel = self.__getOptimizedMetricLabel()<EOL>self._reportMetricLabels = matchPatterns(self._reportKeyPatterns,<EOL>self._getMetricLabels())<EOL>self._periodic = self._initPeriodicActivities()<EOL>numIters = self._modelControl.get('<STR_LIT>', -<NUM_LIT:1>)<EOL>learningOffAt = None<EOL>iterationCountInferOnly = self._modelControl.get('<STR_LIT>', <NUM_LIT:0>)<EOL>if iterationCountInferOnly == -<NUM_LIT:1>:<EOL><INDENT>self._model.disableLearning()<EOL><DEDENT>elif iterationCountInferOnly > <NUM_LIT:0>:<EOL><INDENT>assert numIters > iterationCountInferOnly, "<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>learningOffAt = numIters - iterationCountInferOnly<EOL><DEDENT>self.__runTaskMainLoop(numIters, learningOffAt=learningOffAt)<EOL>self._finalize()<EOL>return (self._cmpReason, None)<EOL>

Runs the OPF Model

        Parameters:
        -------------------------------------------------------------------------
        retval:  (completionReason, completionMsg)
                  where completionReason is one of the ClientJobsDAO.CMPL_REASON_XXX
                    equates.


def __runTaskMainLoop(self, numIters, learningOffAt=None):

<EOL>self._model.resetSequenceStates()<EOL>self._currentRecordIndex = -<NUM_LIT:1><EOL>while True:<EOL><INDENT>if self._isKilled:<EOL><INDENT>break<EOL><DEDENT>if self._isCanceled:<EOL><INDENT>break<EOL><DEDENT>if self._isInterrupted.isSet():<EOL><INDENT>self.__setAsOrphaned()<EOL>break<EOL><DEDENT>if self._isMature:<EOL><INDENT>if not self._isBestModel:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED<EOL>break<EOL><DEDENT>else:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_EOF<EOL><DEDENT>if learningOffAt is not Noneand self._currentRecordIndex == learningOffAt:<EOL><INDENT>self._model.disableLearning()<EOL><DEDENT><DEDENT>try:<EOL><INDENT>inputRecord = self._inputSource.getNextRecordDict()<EOL>if self._currentRecordIndex < <NUM_LIT:0>:<EOL><INDENT>self._inputSource.setTimeout(<NUM_LIT:10>)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>raise utils.JobFailException(ErrorCodes.streamReading, str(e.args),<EOL>traceback.format_exc())<EOL><DEDENT>if inputRecord is None:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_EOF<EOL>break<EOL><DEDENT>if inputRecord:<EOL><INDENT>self._currentRecordIndex += <NUM_LIT:1><EOL>result = self._model.run(inputRecord=inputRecord)<EOL>result.metrics = self.__metricMgr.update(result)<EOL>if not result.metrics:<EOL><INDENT>result.metrics = self.__metricMgr.getMetrics()<EOL><DEDENT>if InferenceElement.encodings in result.inferences:<EOL><INDENT>result.inferences.pop(InferenceElement.encodings)<EOL><DEDENT>result.sensorInput.dataEncodings = None<EOL>self._writePrediction(result)<EOL>self._periodic.tick()<EOL>if numIters >= <NUM_LIT:0> and self._currentRecordIndex >= numIters-<NUM_LIT:1>:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" %<EOL>inputRecord)<EOL><DEDENT><DEDENT>

Main loop of the OPF Model Runner.

        Parameters:
        -----------------------------------------------------------------------

        recordIterator:    Iterator for counting number of records (see _runTask)
        learningOffAt:     If not None, learning is turned off when we reach this
                            iteration number


self._logger.info(<EOL>"<STR_LIT>",<EOL>self._modelID, self._currentRecordIndex + <NUM_LIT:1>)<EOL>self._updateModelDBResults()<EOL>if not self._isKilled:<EOL><INDENT>self.__updateJobResults()<EOL><DEDENT>else:<EOL><INDENT>self.__deleteOutputCache(self._modelID)<EOL><DEDENT>if self._predictionLogger:<EOL><INDENT>self._predictionLogger.close()<EOL><DEDENT>if self._inputSource: <EOL><INDENT>self._inputSource.close()<EOL><DEDENT>

Run final activities after a model has run. These include recording and
        logging the final score


if self._model is None or self._modelCheckpointGUID is None:<EOL><INDENT>return<EOL><DEDENT>if self._predictionLogger is None:<EOL><INDENT>self._createPredictionLogger()<EOL><DEDENT>predictions = io.StringIO()<EOL>self._predictionLogger.checkpoint(<EOL>checkpointSink=predictions,<EOL>maxRows=int(Configuration.get('<STR_LIT>')))<EOL>self._model.save(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))<EOL>self._jobsDAO.modelSetFields(modelID,<EOL>{'<STR_LIT>':str(self._modelCheckpointGUID)},<EOL>ignoreUnchanged=True)<EOL>self._logger.info("<STR_LIT>"<EOL>"<STR_LIT>", self._modelID, checkpointID)<EOL>return<EOL>

Create a checkpoint from the current model, and store it in a dir named
        after checkpoint GUID, and finally store the GUID in the Models DB


Delete the stored checkpoint for the specified modelID. This function is
called if the current model is now the best model, making the old model's
checkpoint obsolete

Parameters:
-----------------------------------------------------------------------
modelID:      The modelID for the checkpoint to delete. This is NOT the
              unique checkpointID


<EOL>self._predictionLogger = BasicPredictionLogger(<EOL>fields=self._model.getFieldInfo(),<EOL>experimentDir=self._experimentDir,<EOL>label = "<STR_LIT>",<EOL>inferenceType=self._model.getInferenceType())<EOL>if self.__loggedMetricPatterns:<EOL><INDENT>metricLabels = self.__metricMgr.getMetricLabels()<EOL>loggedMetrics = matchPatterns(self.__loggedMetricPatterns, metricLabels)<EOL>self._predictionLogger.setLoggedMetrics(loggedMetrics)<EOL><DEDENT>


matchingKeys = matchPatterns([self._optimizeKeyPattern],<EOL>self._getMetricLabels())<EOL>if len(matchingKeys) == <NUM_LIT:0>:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>" %(self._optimizeKeyPattern, self._getMetricLabels()))<EOL><DEDENT>elif len(matchingKeys) > <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>" % (self._optimizeKeyPattern, matchingKeys))<EOL><DEDENT>return matchingKeys[<NUM_LIT:0>]<EOL>

Get the label for the metric being optimized. This function also caches
        the label in the instance variable self._optimizedMetricLabel

        Parameters:
        -----------------------------------------------------------------------
        metricLabels:   A sequence of all the labels being computed for this model

        Returns:        The label for the metric being optmized over


return self.__metricMgr.getMetricLabels()<EOL>

Returns:  A list of labels that correspond to metrics being computed


fieldStats = dict()<EOL>fieldNames = self._inputSource.getFieldNames()<EOL>for field in fieldNames:<EOL><INDENT>curStats = dict()<EOL>curStats['<STR_LIT>'] = self._inputSource.getFieldMin(field)<EOL>curStats['<STR_LIT>'] = self._inputSource.getFieldMax(field)<EOL>fieldStats[field] = curStats<EOL><DEDENT>return fieldStats<EOL>

Method which returns a dictionary of field statistics received from the
input source.

Returns:

  fieldStats: dict of dicts where the first level is the field name and
    the second level is the statistic. ie. fieldStats['pounds']['min']


return self.__metricMgr.getMetrics()<EOL>

Protected function that can be overriden by subclasses. Its main purpose
        is to allow the the OPFDummyModelRunner to override this with deterministic
        values

        Returns: All the metrics being computed for this model


<EOL>metrics = self._getMetrics()<EOL>reportDict = dict([(k,metrics[k]) for k in self._reportMetricLabels])<EOL>metrics = self._getMetrics()<EOL>optimizeDict = dict()<EOL>if self._optimizeKeyPattern is not None:<EOL><INDENT>optimizeDict[self._optimizedMetricLabel] =metrics[self._optimizedMetricLabel]<EOL><DEDENT>results = json.dumps((metrics , optimizeDict))<EOL>self._jobsDAO.modelUpdateResults(self._modelID,  results=results,<EOL>metricValue=list(optimizeDict.values())[<NUM_LIT:0>],<EOL>numRecords=(self._currentRecordIndex + <NUM_LIT:1>))<EOL>self._logger.debug(<EOL>"<STR_LIT>" %(self._modelID, self._currentRecordIndex + <NUM_LIT:1>, results))<EOL>return<EOL>

Retrieves the current results and updates the model's record in
        the Model database.


if self._isBestModelStored and not self._isBestModel:<EOL><INDENT>return<EOL><DEDENT>while True:<EOL><INDENT>jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobResultsStr is None:<EOL><INDENT>jobResults = {}<EOL><DEDENT>else:<EOL><INDENT>self._isBestModelStored = True<EOL>if not self._isBestModel:<EOL><INDENT>return<EOL><DEDENT>jobResults = json.loads(jobResultsStr)<EOL><DEDENT>bestModel = jobResults.get('<STR_LIT>', None)<EOL>bestMetric = jobResults.get('<STR_LIT>', None)<EOL>isSaved = jobResults.get('<STR_LIT>', False)<EOL>if (bestModel is not None) and (self._modelID != bestModel):<EOL><INDENT>self._isBestModel = False<EOL>return<EOL><DEDENT>self.__flushPredictionCache()<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>metrics = self._getMetrics()<EOL>jobResults['<STR_LIT>'] = self._modelID<EOL>jobResults['<STR_LIT>'] = metrics[self._optimizedMetricLabel]<EOL>jobResults['<STR_LIT>'] = metrics<EOL>jobResults['<STR_LIT>'] = False<EOL>newResults = json.dumps(jobResults)<EOL>isUpdated = self._jobsDAO.jobSetFieldIfEqual(self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=jobResultsStr,<EOL>newValue=newResults)<EOL>if isUpdated or (not isUpdated and newResults==jobResultsStr):<EOL><INDENT>self._isBestModel = True<EOL>break<EOL><DEDENT><DEDENT>

Periodic check to see if this is the best model. This should only have an
effect if this is the *first* model to report its progress


jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobResultsStr is None:<EOL><INDENT>jobResults = {}<EOL><DEDENT>else:<EOL><INDENT>jobResults = json.loads(jobResultsStr)<EOL><DEDENT>isSaved = jobResults.get('<STR_LIT>', False)<EOL>bestMetric = jobResults.get('<STR_LIT>', None)<EOL>currentMetric = self._getMetrics()[self._optimizedMetricLabel]<EOL>self._isBestModel = (not isSaved)or (currentMetric < bestMetric)<EOL>return self._isBestModel, jobResults, jobResultsStr<EOL>

Reads the current "best model" for the job and returns whether or not the
current model is better than the "best model" stored for the job

Returns: (isBetter, storedBest, origResultsStr)

isBetter:
  True if the current model is better than the stored "best model"
storedResults:
  A dict of the currently stored results in the jobs table record
origResultsStr:
  The json-encoded string that currently resides in the "results" field
  of the jobs record (used to create atomicity)


isSaved = False<EOL>while True:<EOL><INDENT>self._isBestModel, jobResults, jobResultsStr =self.__checkIfBestCompletedModel()<EOL>if self._isBestModel:<EOL><INDENT>if not isSaved:<EOL><INDENT>self.__flushPredictionCache()<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>self.__createModelCheckpoint()<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>isSaved = True<EOL><DEDENT>prevBest = jobResults.get('<STR_LIT>', None)<EOL>prevWasSaved = jobResults.get('<STR_LIT>', False)<EOL>if prevBest == self._modelID:<EOL><INDENT>assert not prevWasSaved<EOL><DEDENT>metrics = self._getMetrics()<EOL>jobResults['<STR_LIT>'] = self._modelID<EOL>jobResults['<STR_LIT>'] = metrics[self._optimizedMetricLabel]<EOL>jobResults['<STR_LIT>'] = metrics<EOL>jobResults['<STR_LIT>'] = True<EOL>isUpdated = self._jobsDAO.jobSetFieldIfEqual(self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=jobResultsStr,<EOL>newValue=json.dumps(jobResults))<EOL>if isUpdated:<EOL><INDENT>if prevWasSaved:<EOL><INDENT>self.__deleteOutputCache(prevBest)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>self.__deleteModelCheckpoint(prevBest)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL><DEDENT>self._logger.info("<STR_LIT>", self._modelID)<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.__deleteOutputCache(self._modelID)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>self.__deleteModelCheckpoint(self._modelID)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>break<EOL><DEDENT><DEDENT>

Check if this is the best model
If so:
  1) Write it's checkpoint
  2) Record this model as the best
  3) Delete the previous best's output cache
Otherwise:
  1) Delete our output cache


self.__predictionCache.append(result)<EOL>if self._isBestModel:<EOL><INDENT>self.__flushPredictionCache()<EOL><DEDENT>

Writes the results of one iteration of a model. The results are written to
this ModelRunner's in-memory cache unless this model is the "best model" for
the job. If this model is the "best model", the predictions are written out
to a permanent store via a prediction output stream instance

Parameters:
-----------------------------------------------------------------------
result:      A opf_utils.ModelResult object, which contains the input and
              output for this iteration


<EOL>self._jobsDAO.modelUpdateResults(self._modelID)<EOL>

This callback is called by self.__predictionLogger.writeRecords()
        between each batch of records it writes. It gives us a chance to say that
        the model is 'still alive' during long write operations.


if not self.__predictionCache:<EOL><INDENT>return<EOL><DEDENT>if self._predictionLogger is None:<EOL><INDENT>self._createPredictionLogger()<EOL><DEDENT>startTime = time.time()<EOL>self._predictionLogger.writeRecords(self.__predictionCache,<EOL>progressCB=self.__writeRecordsCallback)<EOL>self._logger.info("<STR_LIT>",<EOL>len(self.__predictionCache), time.time() - startTime)<EOL>self.__predictionCache.clear()<EOL>

Writes the contents of this model's in-memory prediction cache to a permanent
store via the prediction output stream instance


<EOL>if modelID == self._modelID and self._predictionLogger is not None:<EOL><INDENT>self._predictionLogger.close()<EOL>del self.__predictionCache<EOL>self._predictionLogger = None<EOL>self.__predictionCache = None<EOL><DEDENT>

Delete's the output cache associated with the given modelID. This actually
clears up the resources associated with the cache, rather than deleting al
the records in the cache

Parameters:
-----------------------------------------------------------------------
modelID:      The id of the model whose output cache is being deleted


Creates and returns a PeriodicActivityMgr instance initialized with
        our periodic activities

        Parameters:
        -------------------------------------------------------------------------
        retval:             a PeriodicActivityMgr instance


<EOL>print("<STR_LIT>", file=sys.stderr)<EOL>jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobCancel:<EOL><INDENT>self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED<EOL>self._isCanceled = True<EOL>self._logger.info("<STR_LIT>",<EOL>self._modelID, self._jobID)<EOL><DEDENT>else:<EOL><INDENT>stopReason = self._jobsDAO.modelsGetFields(self._modelID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if stopReason is None:<EOL><INDENT>pass<EOL><DEDENT>elif stopReason == ClientJobsDAO.STOP_REASON_KILLED:<EOL><INDENT>self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED<EOL>self._isKilled = True<EOL>self._logger.info("<STR_LIT>",<EOL>self._modelID)<EOL><DEDENT>elif stopReason == ClientJobsDAO.STOP_REASON_STOPPED:<EOL><INDENT>self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED<EOL>self._isCanceled = True<EOL>self._logger.info("<STR_LIT>", self._modelID)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError ("<STR_LIT>" % (stopReason))<EOL><DEDENT><DEDENT>

Check if the cancelation flag has been set for this model
        in the Model DB


if self._currentRecordIndex+<NUM_LIT:1> < self._MIN_RECORDS_TO_BE_BEST:<EOL><INDENT>return<EOL><DEDENT>if self._isMature:<EOL><INDENT>return<EOL><DEDENT>metric = self._getMetrics()[self._optimizedMetricLabel]<EOL>self._metricRegression.addPoint(x=self._currentRecordIndex, y=metric)<EOL><INDENT>Perform a linear regression to see if the error is leveled off<EOL><DEDENT>pctChange, absPctChange = self._metricRegression.getPctChanges()<EOL>if pctChange  is not None and absPctChange <= self._MATURITY_MAX_CHANGE:<EOL><INDENT>self._jobsDAO.modelSetFields(self._modelID,<EOL>{'<STR_LIT>':True})<EOL>self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED<EOL>self._isMature = True<EOL>self._logger.info("<STR_LIT>""<STR_LIT>""<STR_LIT>",self._modelID, pctChange,<EOL>self._MATURITY_NUM_POINTS,<EOL>self._metricRegression._window)<EOL><DEDENT>

Save the current metric value and see if the model's performance has
        'leveled off.' We do this by looking at some number of previous number of
        recordings


def handleWarningSignal(self, signum, frame):

Handles a "warning signal" from the scheduler. This is received when the
scheduler is about to kill the the current process so that the worker can be
allocated to another job.

Right now, this function just sets the current model to the "Orphaned" state
in the models table so that another worker can eventually re-run this model

Parameters:
-----------------------------------------------------------------------


cmplReason = ClientJobsDAO.CMPL_REASON_ORPHAN<EOL>cmplMessage = "<STR_LIT>"<EOL>self._jobsDAO.modelSetCompleted(self._modelID, cmplReason, cmplMessage)<EOL>

Sets the current model as orphaned. This is called when the scheduler is
about to kill the process to reallocate the worker to a different process.


return InferenceElement.__inferenceInputMap.get(inferenceElement, None)<EOL>

Get the sensor input element that corresponds to the given inference
        element. This is mainly used for metrics and prediction logging


if InferenceElement.__temporalInferenceElements is None:<EOL><INDENT>InferenceElement.__temporalInferenceElements =set([InferenceElement.prediction])<EOL><DEDENT>return inferenceElement in InferenceElement.__temporalInferenceElements<EOL>

Returns True if the inference from this timestep is predicted the input
        for the NEXT timestep.

        NOTE: This should only be checked IF THE MODEL'S INFERENCE TYPE IS ALSO
        TEMPORAL. That is, a temporal model CAN have non-temporal inference elements,
        but a non-temporal model CANNOT have temporal inference elements


<EOL>if inferenceElement in (InferenceElement.prediction,<EOL>InferenceElement.encodings):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if inferenceElement in (InferenceElement.anomalyScore,<EOL>InferenceElement.anomalyLabel,<EOL>InferenceElement.classification,<EOL>InferenceElement.classConfidences):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>if inferenceElement in (InferenceElement.multiStepPredictions,<EOL>InferenceElement.multiStepBestPredictions):<EOL><INDENT>return int(key)<EOL><DEDENT>return <NUM_LIT:0><EOL>

Returns the number of records that elapse between when an inference is
        made and when the corresponding input record will appear. For example, a
        multistep prediction for 3 timesteps out will have a delay of 3

        Parameters:
        -----------------------------------------------------------------------

        inferenceElement:   The InferenceElement value being delayed
        key:                If the inference is a dictionary type, this specifies
                            key for the sub-inference that is being delayed


maxDelay = <NUM_LIT:0><EOL>for inferenceElement, inference in inferences.iteritems():<EOL><INDENT>if isinstance(inference, dict):<EOL><INDENT>for key in inference.iterkeys():<EOL><INDENT>maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement,<EOL>key),<EOL>maxDelay)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement),<EOL>maxDelay)<EOL><DEDENT><DEDENT>return maxDelay<EOL>

Returns the maximum delay for the InferenceElements in the inference
dictionary

Parameters:
-----------------------------------------------------------------------
inferences:   A dictionary where the keys are InferenceElements


if InferenceType.__temporalInferenceTypes is None:<EOL><INDENT>InferenceType.__temporalInferenceTypes =set([InferenceType.TemporalNextStep,<EOL>InferenceType.TemporalClassification,<EOL>InferenceType.TemporalAnomaly,<EOL>InferenceType.TemporalMultiStep,<EOL>InferenceType.NontemporalMultiStep])<EOL><DEDENT>return inferenceType in InferenceType.__temporalInferenceTypes<EOL>

Returns True if the inference type is 'temporal', i.e. requires a
        temporal memory in the network.


def __init__(self,  jobID, jobsDAO, logLevel = None):

self._jobID = jobID<EOL>self._cjDB = jobsDAO<EOL>self._lastUpdateAttemptTime = <NUM_LIT:0><EOL>initLogging(verbose = True)<EOL>self.logger = logging.getLogger("<STR_LIT:.>".join( ['<STR_LIT>',<EOL>self.__class__.__module__, self.__class__.__name__]))<EOL>if logLevel is not None:<EOL><INDENT>self.logger.setLevel(logLevel)<EOL><DEDENT>self.logger.info("<STR_LIT>" % str(jobID))<EOL>


updateInterval = time.time() - self._lastUpdateAttemptTime<EOL>if updateInterval < self._MIN_UPDATE_INTERVAL and not forceUpdate:<EOL><INDENT>return<EOL><DEDENT>self.logger.info("<STR_LIT>""<STR_LIT>"%(self._jobID,<EOL>time.time(),<EOL>self._lastUpdateAttemptTime))<EOL>timestampUpdated = self._cjDB.jobUpdateSelectionSweep(self._jobID,<EOL>self._MIN_UPDATE_INTERVAL)<EOL>if not timestampUpdated:<EOL><INDENT>self.logger.info("<STR_LIT>""<STR_LIT>"%(self._jobID, self._lastUpdateAttemptTime))<EOL>if not forceUpdate:<EOL><INDENT>return<EOL><DEDENT><DEDENT>self._lastUpdateAttemptTime = time.time()<EOL>self.logger.info("<STR_LIT>"%(self._jobID, self._lastUpdateAttemptTime))<EOL>minUpdateRecords = self._MIN_UPDATE_THRESHOLD<EOL>jobResults = self._getJobResults()<EOL>if forceUpdate or jobResults is None:<EOL><INDENT>minUpdateRecords = <NUM_LIT:0><EOL><DEDENT>candidateIDs, bestMetric = self._cjDB.modelsGetCandidates(self._jobID, minUpdateRecords)<EOL>self.logger.info("<STR_LIT>"%(candidateIDs, bestMetric, self._jobID))<EOL>if len(candidateIDs) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>self._jobUpdateCandidate(candidateIDs[<NUM_LIT:0>], bestMetric, results=jobResults)<EOL>

Chooses the best model for a given job.

        Parameters
        -----------------------------------------------------------------------
        forceUpdate:  (True/False). If True, the update will ignore all the
                      restrictions on the minimum time to update and the minimum
                      number of records to update. This should typically only be
                      set to true if the model has completed running


str = getCopyrightHead() +"<EOL>his file defines parameters for a prediction experiment.<EOL><INDENT>IMPORTANT!!!<EOL><DEDENT>is params file is dynamically generated by the RunExperimentPermutations<EOL>ript. Any changes made manually will be over-written the next time<EOL>nExperimentPermutations is run!!!<EOL><INDENT>nupic.frameworks.opf.exp_description_helpers import importBaseDescription<EOL><DEDENT>e sub-experiment configuration<EOL>ig ={<EOL>return str<EOL>

This is the first portion of every sub-experiment params file we generate. Between
the head and the tail are the experiment specific options.


str ="<EOL>= importBaseDescription('<STR_LIT>', config)<EOL>ls().update(mod.__dict__)<EOL>return str<EOL>

This is the tail of every params file we generate. Between the head and the tail
are the experiment specific options.


def _appendReportKeys(keys, prefix, results):

allKeys = list(results.keys())<EOL>allKeys.sort()<EOL>for key in allKeys:<EOL><INDENT>if hasattr(results[key], '<STR_LIT>'):<EOL><INDENT>_appendReportKeys(keys, "<STR_LIT>" % (prefix, key), results[key])<EOL><DEDENT>else:<EOL><INDENT>keys.add("<STR_LIT>" % (prefix, key))<EOL><DEDENT><DEDENT>

Generate a set of possible report keys for an experiment's results.
A report key is a string of key names separated by colons, each key being one
level deeper into the experiment results dict. For example, 'key1:key2'.

This routine is called recursively to build keys that are multiple levels
deep from the results dict.

Parameters:
-----------------------------------------------------------
keys:         Set of report keys accumulated so far
prefix:       prefix formed so far, this is the colon separated list of key
                names that led up to the dict passed in results
results:      dictionary of results at this level.


matchingReportKeys = []<EOL>for keyRE in reportKeyREs:<EOL><INDENT>matchObj = re.compile(keyRE)<EOL>found = False<EOL>for keyName in allReportKeys:<EOL><INDENT>match = matchObj.match(keyName)<EOL>if match and match.end() == len(keyName):<EOL><INDENT>matchingReportKeys.append(keyName)<EOL>found = True<EOL><DEDENT><DEDENT>if not found:<EOL><INDENT>raise _BadKeyError(keyRE)<EOL><DEDENT><DEDENT>return matchingReportKeys<EOL>

Extract all items from the 'allKeys' list whose key matches one of the regular
expressions passed in 'reportKeys'.

Parameters:
----------------------------------------------------------------------------
reportKeyREs:     List of regular expressions
allReportKeys:    List of all keys

retval:         list of keys from allReportKeys that match the regular expressions
                  in 'reportKeyREs'
                If an invalid regular expression was included in 'reportKeys',
                  then BadKeyError() is raised


subKeys = itemName.split('<STR_LIT::>')<EOL>subResults = results<EOL>for subKey in subKeys:<EOL><INDENT>subResults = subResults[subKey]<EOL><DEDENT>return subResults<EOL>

Get a specific item by name out of the results dict.

The format of itemName is a string of dictionary keys separated by colons,
each key being one level deeper into the results dict. For example,
'key1:key2' would fetch results['key1']['key2'].

If itemName is not found in results, then None is returned


def filterResults(allResults, reportKeys, optimizeKey=None):

<EOL>optimizeDict = dict()<EOL>allReportKeys = set()<EOL>_appendReportKeys(keys=allReportKeys, prefix='<STR_LIT>', results=allResults)<EOL>matchingKeys = _matchReportKeys(reportKeys, allReportKeys)<EOL>reportDict = dict()<EOL>for keyName in matchingKeys:<EOL><INDENT>value = _getReportItem(keyName, allResults)<EOL>reportDict[keyName] = value<EOL><DEDENT>if optimizeKey is not None:<EOL><INDENT>matchingKeys = _matchReportKeys([optimizeKey], allReportKeys)<EOL>if len(matchingKeys) == <NUM_LIT:0>:<EOL><INDENT>raise _BadKeyError(optimizeKey)<EOL><DEDENT>elif len(matchingKeys) > <NUM_LIT:1>:<EOL><INDENT>raise _BadOptimizeKeyError(optimizeKey, matchingKeys)<EOL><DEDENT>optimizeKeyFullName = matchingKeys[<NUM_LIT:0>]<EOL>value = _getReportItem(optimizeKeyFullName, allResults)<EOL>optimizeDict[optimizeKeyFullName] = value<EOL>reportDict[optimizeKeyFullName] = value<EOL><DEDENT>return(reportDict, optimizeDict)<EOL>

Given the complete set of results generated by an experiment (passed in
    'results'), filter out and return only the ones the caller wants, as
    specified through 'reportKeys' and 'optimizeKey'.

    A report key is a string of key names separated by colons, each key being one
    level deeper into the experiment results dict. For example, 'key1:key2'.

    Parameters:
    -------------------------------------------------------------------------
    results:             dict of all results generated by an experiment
    reportKeys:          list of items from the results dict to include in
                         the report. These can be regular expressions.
    optimizeKey:         Which report item, if any, we will be optimizing for. This can
                         also be a regular expression, but is an error if it matches
                         more than one key from the experiment's results.
    retval:  (reportDict, optimizeDict)
                reportDict: a dictionary of the metrics named by desiredReportKeys
                optimizeDict: A dictionary containing 1 item: the full name and
                      value of the metric identified by the optimizeKey


assert type(string) in (str,)<EOL>return pprint.pformat(string)<EOL>

string:   input string (ascii or unicode)

Returns:  a quoted string with characters that are represented in python via
          escape sequences converted to those escape sequences


def _handleModelRunnerException(jobID, modelID, jobsDAO, experimentDir, logger,<EOL>e):

msg = io.StringIO()<EOL>print("<STR_LIT>" % (<EOL>modelID, e, type(e)), file=msg)<EOL>traceback.print_exc(None, msg)<EOL>completionReason = jobsDAO.CMPL_REASON_ERROR<EOL>completionMsg = msg.getvalue()<EOL>logger.error(completionMsg)<EOL>if type(e) is not InvalidConnectionException:<EOL><INDENT>jobsDAO.modelUpdateResults(modelID,  results=None, numRecords=<NUM_LIT:0>)<EOL><DEDENT>if type(e) == JobFailException:<EOL><INDENT>workerCmpReason = jobsDAO.jobGetFields(jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:<EOL><INDENT>jobsDAO.jobSetFields(jobID, fields=dict(<EOL>cancel=True,<EOL>workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,<EOL>workerCompletionMsg = "<STR_LIT>".join(str(i) for i in e.args)),<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL><DEDENT><DEDENT>return (completionReason, completionMsg)<EOL>

Perform standard handling of an exception that occurs while running
    a model.

    Parameters:
    -------------------------------------------------------------------------
    jobID:                ID for this hypersearch job in the jobs table
    modelID:              model ID
    jobsDAO:              ClientJobsDAO instance
    experimentDir:        directory containing the experiment
    logger:               the logger to use
    e:                    the exception that occurred
    retval:               (completionReason, completionMsg)