def update(self, modelID, modelParams, modelParamsHash, metricResult,<EOL>completed, completionReason, matured, numRecords):

<EOL>assert (modelParamsHash is not None)<EOL>if completed:<EOL><INDENT>matured = True<EOL><DEDENT>if metricResult is not None and matured andcompletionReason in [ClientJobsDAO.CMPL_REASON_EOF,<EOL>ClientJobsDAO.CMPL_REASON_STOPPED]:<EOL><INDENT>if self._hsObj._maximize:<EOL><INDENT>errScore = -<NUM_LIT:1> * metricResult<EOL><DEDENT>else:<EOL><INDENT>errScore = metricResult<EOL><DEDENT>if errScore < self._bestResult:<EOL><INDENT>self._bestResult = errScore<EOL>self._bestModelID = modelID<EOL>self._hsObj.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (len(self._allResults), self._bestResult,<EOL>self._bestModelID))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>errScore = numpy.inf<EOL><DEDENT>if completed and completionReason in [ClientJobsDAO.CMPL_REASON_ORPHAN]:<EOL><INDENT>errScore = numpy.inf<EOL>hidden = True<EOL><DEDENT>else:<EOL><INDENT>hidden = False<EOL><DEDENT>if completed:<EOL><INDENT>self._completedModels.add(modelID)<EOL>self._numCompletedModels = len(self._completedModels)<EOL>if completionReason == ClientJobsDAO.CMPL_REASON_ERROR:<EOL><INDENT>self._errModels.add(modelID)<EOL>self._numErrModels = len(self._errModels)<EOL><DEDENT><DEDENT>wasHidden = False<EOL>if modelID not in self._modelIDToIdx:<EOL><INDENT>assert (modelParams is not None)<EOL>entry = dict(modelID=modelID, modelParams=modelParams,<EOL>modelParamsHash=modelParamsHash,<EOL>errScore=errScore, completed=completed,<EOL>matured=matured, numRecords=numRecords, hidden=hidden)<EOL>self._allResults.append(entry)<EOL>entryIdx = len(self._allResults) - <NUM_LIT:1><EOL>self._modelIDToIdx[modelID] = entryIdx<EOL>self._paramsHashToIndexes[modelParamsHash] = entryIdx<EOL>swarmId = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>if not hidden:<EOL><INDENT>if swarmId in self._swarmIdToIndexes:<EOL><INDENT>self._swarmIdToIndexes[swarmId].append(entryIdx)<EOL><DEDENT>else:<EOL><INDENT>self._swarmIdToIndexes[swarmId] = [entryIdx]<EOL><DEDENT>genIdx = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>numPsEntry = self._swarmNumParticlesPerGeneration.get(swarmId, [<NUM_LIT:0>])<EOL>while genIdx >= len(numPsEntry):<EOL><INDENT>numPsEntry.append(<NUM_LIT:0>)<EOL><DEDENT>numPsEntry[genIdx] += <NUM_LIT:1><EOL>self._swarmNumParticlesPerGeneration[swarmId] = numPsEntry<EOL><DEDENT><DEDENT>else:<EOL><INDENT>entryIdx = self._modelIDToIdx.get(modelID, None)<EOL>assert (entryIdx is not None)<EOL>entry = self._allResults[entryIdx]<EOL>wasHidden = entry['<STR_LIT>']<EOL>if entry['<STR_LIT>'] != modelParamsHash:<EOL><INDENT>self._paramsHashToIndexes.pop(entry['<STR_LIT>'])<EOL>self._paramsHashToIndexes[modelParamsHash] = entryIdx<EOL>entry['<STR_LIT>'] = modelParamsHash<EOL><DEDENT>modelParams = entry['<STR_LIT>']<EOL>swarmId = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>genIdx = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>if hidden and not wasHidden:<EOL><INDENT>assert (entryIdx in self._swarmIdToIndexes[swarmId])<EOL>self._swarmIdToIndexes[swarmId].remove(entryIdx)<EOL>self._swarmNumParticlesPerGeneration[swarmId][genIdx] -= <NUM_LIT:1><EOL><DEDENT>entry['<STR_LIT>']  = errScore<EOL>entry['<STR_LIT>'] = completed<EOL>entry['<STR_LIT>'] = matured<EOL>entry['<STR_LIT>'] = numRecords<EOL>entry['<STR_LIT>'] = hidden<EOL><DEDENT>particleId = modelParams['<STR_LIT>']['<STR_LIT:id>']<EOL>genIdx = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>if matured and not hidden:<EOL><INDENT>(oldResult, pos) = self._particleBest.get(particleId, (numpy.inf, None))<EOL>if errScore < oldResult:<EOL><INDENT>pos = Particle.getPositionFromState(modelParams['<STR_LIT>'])<EOL>self._particleBest[particleId] = (errScore, pos)<EOL><DEDENT><DEDENT>prevGenIdx = self._particleLatestGenIdx.get(particleId, -<NUM_LIT:1>)<EOL>if not hidden and genIdx > prevGenIdx:<EOL><INDENT>self._particleLatestGenIdx[particleId] = genIdx<EOL><DEDENT>elif hidden and not wasHidden and genIdx == prevGenIdx:<EOL><INDENT>self._particleLatestGenIdx[particleId] = genIdx-<NUM_LIT:1><EOL><DEDENT>if not hidden:<EOL><INDENT>swarmId = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>if not swarmId in self._swarmBestOverall:<EOL><INDENT>self._swarmBestOverall[swarmId] = []<EOL><DEDENT>bestScores = self._swarmBestOverall[swarmId]<EOL>while genIdx >= len(bestScores):<EOL><INDENT>bestScores.append((None, numpy.inf))<EOL><DEDENT>if errScore < bestScores[genIdx][<NUM_LIT:1>]:<EOL><INDENT>bestScores[genIdx] = (modelID, errScore)<EOL><DEDENT><DEDENT>if not hidden:<EOL><INDENT>key = (swarmId, genIdx)<EOL>if not key in self._maturedSwarmGens:<EOL><INDENT>self._modifiedSwarmGens.add(key)<EOL><DEDENT><DEDENT>return errScore<EOL>

Insert a new entry or update an existing one. If this is an update
        of an existing entry, then modelParams will be None

        Parameters:
        --------------------------------------------------------------------
        modelID:       globally unique modelID of this model
        modelParams:    params dict for this model, or None if this is just an update
                        of a model that it already previously reported on.

                        See the comments for the createModels() method for
                        a description of this dict.

        modelParamsHash:  hash of the modelParams dict, generated by the worker
                        that put it into the model database.
        metricResult:   value on the optimizeMetric for this model.
                        May be None if we have no results yet.
        completed:      True if the model has completed evaluation, False if it
                          is still running (and these are online results)
        completionReason: One of the ClientJobsDAO.CMPL_REASON_XXX equates
        matured:        True if this model has matured
        numRecords:     Number of records that have been processed so far by this
                          model.

        retval: Canonicalized result on the optimize metric


return self._numErrModels<EOL>


return list(self._errModels)<EOL>


return self._numCompletedModels<EOL>

Return total number of models that completed.

        Parameters:
        ---------------------------------------------------------------------
        retval:      # if models that completed


entryIdx = self. _paramsHashToIndexes.get(paramsHash, None)<EOL>if entryIdx is not None:<EOL><INDENT>return self._allResults[entryIdx]['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Return the modelID of the model with the given paramsHash, or
        None if not found.

        Parameters:
        ---------------------------------------------------------------------
        paramsHash:  paramsHash to look for
        retval:      modelId, or None if not found


def numModels(self, swarmId=None, includeHidden=False):

<EOL>if includeHidden:<EOL><INDENT>if swarmId is None:<EOL><INDENT>return len(self._allResults)<EOL><DEDENT>else:<EOL><INDENT>return len(self._swarmIdToIndexes.get(swarmId, []))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if swarmId is None:<EOL><INDENT>entries = self._allResults<EOL><DEDENT>else:<EOL><INDENT>entries = [self._allResults[entryIdx]<EOL>for entryIdx in self._swarmIdToIndexes.get(swarmId,[])]<EOL><DEDENT>return len([entry for entry in entries if not entry['<STR_LIT>']])<EOL><DEDENT>

Return the total # of models we have in our database (if swarmId is
        None) or in a specific swarm.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:        A string representation of the sorted list of encoders
                        in this swarm. For example '__address_encoder.__gym_encoder'
        includeHidden:  If False, this will only return the number of models
                        that are not hidden (i.e. orphanned, etc.)
        retval:  numModels


def bestModelIdAndErrScore(self, swarmId=None, genIdx=None):

if swarmId is None:<EOL><INDENT>return (self._bestModelID, self._bestResult)<EOL><DEDENT>else:<EOL><INDENT>if swarmId not in self._swarmBestOverall:<EOL><INDENT>return (None, numpy.inf)<EOL><DEDENT>genScores = self._swarmBestOverall[swarmId]<EOL>bestModelId = None<EOL>bestScore = numpy.inf<EOL>for (i, (modelId, errScore)) in enumerate(genScores):<EOL><INDENT>if genIdx is not None and i > genIdx:<EOL><INDENT>break<EOL><DEDENT>if errScore < bestScore:<EOL><INDENT>bestScore = errScore<EOL>bestModelId = modelId<EOL><DEDENT><DEDENT>return (bestModelId, bestScore)<EOL><DEDENT>

Return the model ID of the model with the best result so far and
        it's score on the optimize metric. If swarm is None, then it returns
        the global best, otherwise it returns the best for the given swarm
        for all generatons up to and including genIdx.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'
        genIdx:   consider the best in all generations up to and including this
                    generation if not None.
        retval:  (modelID, result)


entry = self._allResults[self._modelIDToIdx[modelId]]<EOL>return (entry['<STR_LIT>']['<STR_LIT>'], modelId, entry['<STR_LIT>'],<EOL>entry['<STR_LIT>'], entry['<STR_LIT>'])<EOL>

Return particle info for a specific modelId.

        Parameters:
        ---------------------------------------------------------------------
        modelId:  which model Id

        retval:  (particleState, modelId, errScore, completed, matured)


def getParticleInfos(self, swarmId=None, genIdx=None, completed=None,<EOL>matured=None, lastDescendent=False):

<EOL>if swarmId is not None:<EOL><INDENT>entryIdxs = self._swarmIdToIndexes.get(swarmId, [])<EOL><DEDENT>else:<EOL><INDENT>entryIdxs = list(range(len(self._allResults)))<EOL><DEDENT>if len(entryIdxs) == <NUM_LIT:0>:<EOL><INDENT>return ([], [], [], [], [])<EOL><DEDENT>particleStates = []<EOL>modelIds = []<EOL>errScores = []<EOL>completedFlags = []<EOL>maturedFlags = []<EOL>for idx in entryIdxs:<EOL><INDENT>entry = self._allResults[idx]<EOL>if swarmId is not None:<EOL><INDENT>assert (not entry['<STR_LIT>'])<EOL><DEDENT>modelParams = entry['<STR_LIT>']<EOL>isCompleted = entry['<STR_LIT>']<EOL>isMatured = entry['<STR_LIT>']<EOL>particleState = modelParams['<STR_LIT>']<EOL>particleGenIdx = particleState['<STR_LIT>']<EOL>particleId = particleState['<STR_LIT:id>']<EOL>if genIdx is not None and particleGenIdx != genIdx:<EOL><INDENT>continue<EOL><DEDENT>if completed is not None and (completed != isCompleted):<EOL><INDENT>continue<EOL><DEDENT>if matured is not None and (matured != isMatured):<EOL><INDENT>continue<EOL><DEDENT>if lastDescendentand (self._particleLatestGenIdx[particleId] != particleGenIdx):<EOL><INDENT>continue<EOL><DEDENT>particleStates.append(particleState)<EOL>modelIds.append(entry['<STR_LIT>'])<EOL>errScores.append(entry['<STR_LIT>'])<EOL>completedFlags.append(isCompleted)<EOL>maturedFlags.append(isMatured)<EOL><DEDENT>return (particleStates, modelIds, errScores, completedFlags, maturedFlags)<EOL>

Return a list of particleStates for all particles we know about in
        the given swarm, their model Ids, and metric results.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'

        genIdx:  If not None, only return particles at this specific generation
                      index.

        completed:   If not None, only return particles of the given state (either
                    completed if 'completed' is True, or running if 'completed'
                    is false

        matured:   If not None, only return particles of the given state (either
                    matured if 'matured' is True, or not matured if 'matured'
                    is false. Note that any model which has completed is also
                    considered matured.

        lastDescendent: If True, only return particles that are the last descendent,
                    that is, the highest generation index for a given particle Id

        retval:  (particleStates, modelIds, errScores, completed, matured)
                  particleStates: list of particleStates
                  modelIds: list of modelIds
                  errScores: list of errScores, numpy.inf is plugged in
                                  if we don't have a result yet
                  completed: list of completed booleans
                  matured: list of matured booleans


def getOrphanParticleInfos(self, swarmId, genIdx):

entryIdxs = list(range(len(self._allResults)))<EOL>if len(entryIdxs) == <NUM_LIT:0>:<EOL><INDENT>return ([], [], [], [], [])<EOL><DEDENT>particleStates = []<EOL>modelIds = []<EOL>errScores = []<EOL>completedFlags = []<EOL>maturedFlags = []<EOL>for idx in entryIdxs:<EOL><INDENT>entry = self._allResults[idx]<EOL>if not entry['<STR_LIT>']:<EOL><INDENT>continue<EOL><DEDENT>modelParams = entry['<STR_LIT>']<EOL>if modelParams['<STR_LIT>']['<STR_LIT>'] != swarmId:<EOL><INDENT>continue<EOL><DEDENT>isCompleted = entry['<STR_LIT>']<EOL>isMatured = entry['<STR_LIT>']<EOL>particleState = modelParams['<STR_LIT>']<EOL>particleGenIdx = particleState['<STR_LIT>']<EOL>particleId = particleState['<STR_LIT:id>']<EOL>if genIdx is not None and particleGenIdx != genIdx:<EOL><INDENT>continue<EOL><DEDENT>particleStates.append(particleState)<EOL>modelIds.append(entry['<STR_LIT>'])<EOL>errScores.append(entry['<STR_LIT>'])<EOL>completedFlags.append(isCompleted)<EOL>maturedFlags.append(isMatured)<EOL><DEDENT>return (particleStates, modelIds, errScores, completedFlags, maturedFlags)<EOL>

Return a list of particleStates for all particles in the given
        swarm generation that have been orphaned.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'

        genIdx:  If not None, only return particles at this specific generation
                      index.

        retval:  (particleStates, modelIds, errScores, completed, matured)
                  particleStates: list of particleStates
                  modelIds: list of modelIds
                  errScores: list of errScores, numpy.inf is plugged in
                                  if we don't have a result yet
                  completed: list of completed booleans
                  matured: list of matured booleans


<EOL>result = []<EOL>modifiedSwarmGens = sorted(self._modifiedSwarmGens)<EOL>for key in modifiedSwarmGens:<EOL><INDENT>(swarmId, genIdx) = key<EOL>if key in self._maturedSwarmGens:<EOL><INDENT>self._modifiedSwarmGens.remove(key)<EOL>continue<EOL><DEDENT>if (genIdx >= <NUM_LIT:1>) and not (swarmId, genIdx-<NUM_LIT:1>) in self._maturedSwarmGens:<EOL><INDENT>continue<EOL><DEDENT>(_, _, errScores, completedFlags, maturedFlags) =self.getParticleInfos(swarmId, genIdx)<EOL>maturedFlags = numpy.array(maturedFlags)<EOL>numMatured = maturedFlags.sum()<EOL>if numMatured >= self._hsObj._minParticlesPerSwarmand numMatured == len(maturedFlags):<EOL><INDENT>errScores = numpy.array(errScores)<EOL>bestScore = errScores.min()<EOL>self._maturedSwarmGens.add(key)<EOL>self._modifiedSwarmGens.remove(key)<EOL>result.append((swarmId, genIdx, bestScore))<EOL><DEDENT><DEDENT>return result<EOL>

Return a list of swarm generations that have completed and the
        best (minimal) errScore seen for each of them.

        Parameters:
        ---------------------------------------------------------------------
        retval:  list of tuples. Each tuple is of the form:
                  (swarmId, genIdx, bestErrScore)


def firstNonFullGeneration(self, swarmId, minNumParticles):

if not swarmId in self._swarmNumParticlesPerGeneration:<EOL><INDENT>return None<EOL><DEDENT>numPsPerGen = self._swarmNumParticlesPerGeneration[swarmId]<EOL>numPsPerGen = numpy.array(numPsPerGen)<EOL>firstNonFull = numpy.where(numPsPerGen < minNumParticles)[<NUM_LIT:0>]<EOL>if len(firstNonFull) == <NUM_LIT:0>:<EOL><INDENT>return len(numPsPerGen)<EOL><DEDENT>else:<EOL><INDENT>return firstNonFull[<NUM_LIT:0>]<EOL><DEDENT>

Return the generation index of the first generation in the given
        swarm that does not have numParticles particles in it, either still in the
        running state or completed. This does not include orphaned particles.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'
        minNumParticles: minium number of partices required for a full
                      generation.

        retval:  generation index, or None if no particles at all.


numPsPerGen = self._swarmNumParticlesPerGeneration[swarmId]<EOL>return len(numPsPerGen)-<NUM_LIT:1><EOL>

Return the generation index of the highest generation in the given
        swarm.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'
        retval:  generation index


return self._particleBest.get(particleId, (None, None))<EOL>

Return the best score and position for a given particle. The position
        is given as a dict, with varName:varPosition items in it.

        Parameters:
        ---------------------------------------------------------------------
        particleId:    which particle
        retval:        (bestResult, bestPosition)


def getResultsPerChoice(self, swarmId, maxGenIdx, varName):

results = dict()<EOL>(allParticles, _, resultErrs, _, _) = self.getParticleInfos(swarmId,<EOL>genIdx=None, matured=True)<EOL>for particleState, resultErr in zip(allParticles, resultErrs):<EOL><INDENT>if maxGenIdx is not None:<EOL><INDENT>if particleState['<STR_LIT>'] > maxGenIdx:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>if resultErr == numpy.inf:<EOL><INDENT>continue<EOL><DEDENT>position = Particle.getPositionFromState(particleState)<EOL>varPosition = position[varName]<EOL>varPositionStr = str(varPosition)<EOL>if varPositionStr in results:<EOL><INDENT>results[varPositionStr][<NUM_LIT:1>].append(resultErr)<EOL><DEDENT>else:<EOL><INDENT>results[varPositionStr] = (varPosition, [resultErr])<EOL><DEDENT><DEDENT>return results<EOL>

Return a dict of the errors obtained on models that were run with
        each value from a PermuteChoice variable.

        For example, if a PermuteChoice variable has the following choices:
          ['a', 'b', 'c']

        The dict will have 3 elements. The keys are the stringified choiceVars,
        and each value is tuple containing (choiceVar, errors) where choiceVar is
        the original form of the choiceVar (before stringification) and errors is
        the list of errors received from models that used the specific choice:
        retval:
          ['a':('a', [0.1, 0.2, 0.3]), 'b':('b', [0.5, 0.1, 0.6]), 'c':('c', [])]

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  swarm Id of the swarm to retrieve info from
        maxGenIdx: max generation index to consider from other models, ignored
                    if None
        varName:  which variable to retrieve

        retval:  list of the errors obtained from each choice.


def __init__(self, searchParams, workerID=None, cjDAO=None, jobID=None,<EOL>logLevel=None):

<EOL>self.logger = logging.getLogger("<STR_LIT:.>".join( ['<STR_LIT>',<EOL>self.__class__.__module__, self.__class__.__name__]))<EOL>if logLevel is not None:<EOL><INDENT>self.logger.setLevel(logLevel)<EOL><DEDENT>random.seed(<NUM_LIT>)<EOL>self._searchParams = searchParams<EOL>self._workerID = workerID<EOL>self._cjDAO = cjDAO<EOL>self._jobID = jobID<EOL>self.logger.info("<STR_LIT>" % (pprint.pformat(<EOL>clippedObj(searchParams))))<EOL>self._createCheckpoints = self._searchParams.get('<STR_LIT>',<EOL>False)<EOL>self._maxModels = self._searchParams.get('<STR_LIT>', None)<EOL>if self._maxModels == -<NUM_LIT:1>:<EOL><INDENT>self._maxModels = None<EOL><DEDENT>self._predictionCacheMaxRecords = self._searchParams.get('<STR_LIT>', None)<EOL>self._speculativeParticles = self._searchParams.get('<STR_LIT>',<EOL>bool(int(Configuration.get(<EOL>'<STR_LIT>'))))<EOL>self._speculativeWaitSecondsMax = float(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._maxBranching= int(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._minFieldContribution= float(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._jobCancelled = False<EOL>if '<STR_LIT>' in self._searchParams:<EOL><INDENT>useTerminators = self._searchParams['<STR_LIT>']<EOL>useTerminators = str(int(useTerminators))<EOL>Configuration.set('<STR_LIT>', useTerminators)<EOL>Configuration.set('<STR_LIT>', useTerminators)<EOL>Configuration.set('<STR_LIT>', useTerminators)<EOL><DEDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>self._maxModels = int(os.environ['<STR_LIT>'])<EOL><DEDENT>self._dummyModel = self._searchParams.get('<STR_LIT>', None)<EOL>self._tempDir = None<EOL>try:<EOL><INDENT>if '<STR_LIT:description>' in self._searchParams:<EOL><INDENT>if ('<STR_LIT>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams):<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>searchParamObj = self._searchParams<EOL>anomalyParams = searchParamObj['<STR_LIT:description>'].get('<STR_LIT>',<EOL>dict())<EOL>if anomalyParams is None:<EOL><INDENT>anomalyParams = dict()<EOL><DEDENT>if (('<STR_LIT>' not in anomalyParams) or<EOL>(anomalyParams['<STR_LIT>'] is None)):<EOL><INDENT>streamDef = self._getStreamDef(searchParamObj['<STR_LIT:description>'])<EOL>from nupic.data.stream_reader import StreamReader<EOL>try:<EOL><INDENT>streamReader = StreamReader(streamDef, isBlocking=False,<EOL>maxTimeout=<NUM_LIT:0>, eofOnTimeout=True)<EOL>anomalyParams['<STR_LIT>'] =streamReader.getDataRowCount()<EOL><DEDENT>except Exception:<EOL><INDENT>anomalyParams['<STR_LIT>'] = None<EOL><DEDENT>self._searchParams['<STR_LIT:description>']['<STR_LIT>'] = anomalyParams<EOL><DEDENT>outDir = self._tempDir = tempfile.mkdtemp()<EOL>expGenerator([<EOL>'<STR_LIT>' % (<EOL>json.dumps(self._searchParams['<STR_LIT:description>'])),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % (outDir)])<EOL>permutationsScript = os.path.join(outDir, '<STR_LIT>')<EOL><DEDENT>elif '<STR_LIT>' in self._searchParams:<EOL><INDENT>if ('<STR_LIT:description>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams):<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>permutationsScript = self._searchParams['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in self._searchParams:<EOL><INDENT>if ('<STR_LIT:description>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams):<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>assert ('<STR_LIT>' in self._searchParams)<EOL>outDir = self._tempDir = tempfile.mkdtemp()<EOL>permutationsScript = os.path.join(outDir, '<STR_LIT>')<EOL>fd = open(permutationsScript, '<STR_LIT:w>')<EOL>fd.write(self._searchParams['<STR_LIT>'])<EOL>fd.close()<EOL>fd = open(os.path.join(outDir, '<STR_LIT>'), '<STR_LIT:w>')<EOL>fd.write(self._searchParams['<STR_LIT>'])<EOL>fd.close()<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._basePath = os.path.dirname(permutationsScript)<EOL>self._baseDescription = open(os.path.join(self._basePath,<EOL>'<STR_LIT>')).read()<EOL>self._baseDescriptionHash = hashlib.md5(self._baseDescription).digest()<EOL>modelDescription, _ = helpers.loadExperiment(self._basePath)<EOL>self._readPermutationsFile(permutationsScript, modelDescription)<EOL>if self._cjDAO is not None:<EOL><INDENT>updated = self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=None,<EOL>newValue = self._baseDescription)<EOL>if updated:<EOL><INDENT>permContents = open(permutationsScript).read()<EOL>self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=None,<EOL>newValue = permContents)<EOL><DEDENT><DEDENT>if self._dummyModelParamsFunc is not None:<EOL><INDENT>if self._dummyModel is None:<EOL><INDENT>self._dummyModel = dict()<EOL><DEDENT><DEDENT>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>msg = io.StringIO()<EOL>print("<STR_LIT>", file=msg)<EOL>info = dict()<EOL>for key in ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>info[key] = getattr(self, key)<EOL><DEDENT>print(pprint.pformat(info), file=msg)<EOL>self.logger.debug(msg.getvalue())<EOL>msg.close()<EOL><DEDENT>self._resultsDB = ResultsDB(self)<EOL>self._swarmTerminator = SwarmTerminator()<EOL>self._hsState = None<EOL>self._maxUniqueModelAttempts = int(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._modelOrphanIntervalSecs = float(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._maxPctErrModels = float(Configuration.get(<EOL>'<STR_LIT>'))<EOL><DEDENT>except:<EOL><INDENT>if self._tempDir is not None:<EOL><INDENT>shutil.rmtree(self._tempDir)<EOL>self._tempDir = None<EOL><DEDENT>raise<EOL><DEDENT>return<EOL>

Instantiate the HyperseachV2 instance.

        Parameters:
        ----------------------------------------------------------------------
        searchParams:   a dict of the job's search parameters. The format is:

          persistentJobGUID:  REQUIRED.
                              Persistent, globally-unique identifier for this job
                              for use in constructing persistent model checkpoint
                              keys. MUST be compatible with S3 key-naming rules, but
                              MUST NOT contain forward slashes. This GUID is
                              expected to retain its global uniqueness across
                              clusters and cluster software updates (unlike the
                              record IDs in the Engine's jobs table, which recycle
                              upon table schema change and software update). In the
                              future, this may also be instrumental for checkpoint
                              garbage collection.

          permutationsPyFilename:
                              OPTIONAL - path to permutations.py file
          permutationsPyContents:
                              OPTIONAL - JSON encoded string with
                                          contents of permutations.py file
          descriptionPyContents:
                              OPTIONAL - JSON encoded string with
                                          contents of base description.py file
          description:        OPTIONAL - JSON description of the search
          createCheckpoints:  OPTIONAL - Whether to create checkpoints
          useTerminators      OPTIONAL - True of False (default config.xml). When set
                                         to False, the model and swarm terminators
                                         are disabled
          maxModels:          OPTIONAL - max # of models to generate
                                        NOTE: This is a deprecated location for this
                                        setting. Now, it should be specified through
                                        the maxModels variable within the permutations
                                        file, or maxModels in the JSON description
          dummyModel:         OPTIONAL - Either (True/False) or a dict of parameters
                                         for a dummy model. If this key is absent,
                                         a real model is trained.
                                         See utils.py/OPFDummyModel runner for the
                                         schema of the dummy parameters
          speculativeParticles OPTIONAL - True or False (default obtained from
                                         nupic.hypersearch.speculative.particles.default
                                         configuration property). See note below.

          NOTE: The caller must provide just ONE of the following to describe the
          hypersearch:
                1.) permutationsPyFilename
            OR  2.) permutationsPyContents & permutationsPyContents
            OR  3.) description

          The schema for the description element can be found at:
           "py/nupic/frameworks/opf/expGenerator/experimentDescriptionSchema.json"

          NOTE about speculativeParticles: If true (not 0), hypersearch workers will
          go ahead and create and run particles in subsequent sprints and
          generations before the current generation or sprint has been completed. If
          false, a worker will wait in a sleep loop until the current generation or
          sprint has finished before choosing the next particle position or going
          into the next sprint. When true, the best model can be found faster, but
          results are less repeatable due to the randomness of when each worker
          completes each particle. This property can be overridden via the
          speculativeParticles element of the Hypersearch job params.

        workerID:   our unique Hypersearch worker ID

        cjDAO:      ClientJobsDB Data Access Object
        jobID:      job ID for this hypersearch job
        logLevel:   override logging level to this value, if not None


<EOL>aggregationPeriod = {<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>}<EOL>aggFunctionsDict = {}<EOL>if '<STR_LIT>' in modelDescription['<STR_LIT>']:<EOL><INDENT>for key in list(aggregationPeriod.keys()):<EOL><INDENT>if key in modelDescription['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>aggregationPeriod[key] = modelDescription['<STR_LIT>']['<STR_LIT>'][key]<EOL><DEDENT><DEDENT>if '<STR_LIT>' in modelDescription['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>for (fieldName, func) in modelDescription['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>aggFunctionsDict[fieldName] = str(func)<EOL><DEDENT><DEDENT><DEDENT>hasAggregation = False<EOL>for v in list(aggregationPeriod.values()):<EOL><INDENT>if v != <NUM_LIT:0>:<EOL><INDENT>hasAggregation = True<EOL>break<EOL><DEDENT><DEDENT>aggFunctionList = list(aggFunctionsDict.items())<EOL>aggregationInfo = dict(aggregationPeriod)<EOL>aggregationInfo['<STR_LIT>'] = aggFunctionList<EOL>streamDef = copy.deepcopy(modelDescription['<STR_LIT>'])<EOL>streamDef['<STR_LIT>'] = copy.deepcopy(aggregationInfo)<EOL>return streamDef<EOL>


self.close()<EOL>return<EOL>

Destructor; NOTE: this is not guaranteed to be called (bugs like
        circular references could prevent it from being called).


if self._tempDir is not None and os.path.isdir(self._tempDir):<EOL><INDENT>self.logger.debug("<STR_LIT>", self._tempDir)<EOL>shutil.rmtree(self._tempDir)<EOL>self._tempDir = None<EOL><DEDENT>return<EOL>


def _readPermutationsFile(self, filename, modelDescription):

<EOL>vars = {}<EOL>permFile = exec(compile(open(filename, "<STR_LIT:rb>").read(), filename, '<STR_LIT>'), globals(), vars)<EOL>self._reportKeys = vars.get('<STR_LIT>', [])<EOL>self._filterFunc = vars.get('<STR_LIT>', None)<EOL>self._dummyModelParamsFunc = vars.get('<STR_LIT>', None)<EOL>self._predictedField = None   <EOL>self._predictedFieldEncoder = None   <EOL>self._fixedFields = None <EOL>self._fastSwarmModelParams = vars.get('<STR_LIT>', None)<EOL>if self._fastSwarmModelParams is not None:<EOL><INDENT>encoders = self._fastSwarmModelParams['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL>self._fixedFields = []<EOL>for fieldName in encoders:<EOL><INDENT>if encoders[fieldName] is not None:<EOL><INDENT>self._fixedFields.append(fieldName)<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in vars:<EOL><INDENT>self._fixedFields = vars['<STR_LIT>']<EOL><DEDENT>self._minParticlesPerSwarm = vars.get('<STR_LIT>')<EOL>if self._minParticlesPerSwarm  == None:<EOL><INDENT>self._minParticlesPerSwarm = Configuration.get(<EOL>'<STR_LIT>')<EOL><DEDENT>self._minParticlesPerSwarm = int(self._minParticlesPerSwarm)<EOL>self._killUselessSwarms = vars.get('<STR_LIT>', True)<EOL>self._inputPredictedField = vars.get("<STR_LIT>", "<STR_LIT:yes>")<EOL>self._tryAll3FieldCombinations = vars.get('<STR_LIT>', False)<EOL>self._tryAll3FieldCombinationsWTimestamps = vars.get(<EOL>'<STR_LIT>', False)<EOL>minFieldContribution = vars.get('<STR_LIT>', None)<EOL>if minFieldContribution is not None:<EOL><INDENT>self._minFieldContribution = minFieldContribution<EOL><DEDENT>maxBranching = vars.get('<STR_LIT>', None)<EOL>if maxBranching is not None:<EOL><INDENT>self._maxBranching = maxBranching<EOL><DEDENT>if '<STR_LIT>' in vars:<EOL><INDENT>self._optimizeKey = vars['<STR_LIT>']<EOL>self._maximize = True<EOL><DEDENT>elif '<STR_LIT>' in vars:<EOL><INDENT>self._optimizeKey = vars['<STR_LIT>']<EOL>self._maximize = False<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>maxModels = vars.get('<STR_LIT>')<EOL>if maxModels is not None:<EOL><INDENT>if self._maxModels is None:<EOL><INDENT>self._maxModels = maxModels<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>inferenceType = modelDescription['<STR_LIT>']['<STR_LIT>']<EOL>if not InferenceType.validate(inferenceType):<EOL><INDENT>raise ValueError("<STR_LIT>" %inferenceType)<EOL><DEDENT>if inferenceType in [InferenceType.TemporalMultiStep,<EOL>InferenceType.NontemporalMultiStep]:<EOL><INDENT>classifierOnlyEncoder = None<EOL>for encoder in list(modelDescription["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"].values()):<EOL><INDENT>if encoder.get("<STR_LIT>", False)and encoder["<STR_LIT>"] == vars.get('<STR_LIT>', None):<EOL><INDENT>classifierOnlyEncoder = encoder<EOL>break<EOL><DEDENT><DEDENT>if classifierOnlyEncoder is None or self._inputPredictedField=="<STR_LIT:yes>":<EOL><INDENT>self._searchType = HsSearchType.legacyTemporal<EOL><DEDENT>else:<EOL><INDENT>self._searchType = HsSearchType.temporal<EOL><DEDENT><DEDENT>elif inferenceType in [InferenceType.TemporalNextStep,<EOL>InferenceType.TemporalAnomaly]:<EOL><INDENT>self._searchType = HsSearchType.legacyTemporal<EOL><DEDENT>elif inferenceType in (InferenceType.TemporalClassification,<EOL>InferenceType.NontemporalClassification):<EOL><INDENT>self._searchType = HsSearchType.classification<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % inferenceType)<EOL><DEDENT>self._predictedField = vars.get('<STR_LIT>', None)<EOL>if self._predictedField is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % filename)<EOL><DEDENT>if '<STR_LIT>' not in vars:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % filename)<EOL><DEDENT>if not isinstance(vars['<STR_LIT>'], dict):<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._encoderNames = []<EOL>self._permutations = vars['<STR_LIT>']<EOL>self._flattenedPermutations = dict()<EOL>def _flattenPermutations(value, keys):<EOL><INDENT>if '<STR_LIT::>' in keys[-<NUM_LIT:1>]:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>flatKey = _flattenKeys(keys)<EOL>if isinstance(value, PermuteEncoder):<EOL><INDENT>self._encoderNames.append(flatKey)<EOL>if value.fieldName == self._predictedField:<EOL><INDENT>self._predictedFieldEncoder = flatKey<EOL><DEDENT>for encKey, encValue in value.kwArgs.items():<EOL><INDENT>if isinstance(encValue, PermuteVariable):<EOL><INDENT>self._flattenedPermutations['<STR_LIT>' % (flatKey, encKey)] = encValue<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(value, PermuteVariable):<EOL><INDENT>self._flattenedPermutations[flatKey] = value<EOL><DEDENT>else:<EOL><INDENT>if isinstance(value, PermuteVariable):<EOL><INDENT>self._flattenedPermutations[key] = value<EOL><DEDENT><DEDENT><DEDENT>rApply(self._permutations, _flattenPermutations)<EOL>

Read the permutations file and initialize the following member variables:
    _predictedField: field name of the field we are trying to
      predict
    _permutations: Dict containing the full permutations dictionary.
    _flattenedPermutations: Dict containing the flattened version of
      _permutations. The keys leading to the value in the dict are joined
      with a period to create the new key and permute variables within
      encoders are pulled out of the encoder.
    _encoderNames: keys from self._permutations of only the encoder
      variables.
    _reportKeys:   The 'report' list from the permutations file.
      This is a list of the items from each experiment's pickled
      results file that should be included in the final report. The
      format of each item is a string of key names separated by colons,
      each key being one level deeper into the experiment results
      dict. For example, 'key1:key2'.
    _filterFunc: a user-supplied function that can be used to
      filter out specific permutation combinations.
    _optimizeKey: which report key to optimize for
    _maximize: True if we should try and maximize the optimizeKey
      metric. False if we should minimize it.
    _dummyModelParamsFunc: a user-supplied function that can be used to
      artificially generate HTMPredictionModel results. When supplied,
      the model is not actually run through the OPF, but instead is run
      through a "Dummy Model" (nupic.swarming.ModelRunner.
      OPFDummyModelRunner). This function returns the params dict used
      to control various options in the dummy model (the returned metric,
      the execution time, etc.). This is used for hypersearch algorithm
      development.

Parameters:
---------------------------------------------------------
filename:     Name of permutations file
retval:       None


return -<NUM_LIT:1><EOL>

Computes the number of models that are expected to complete as part of
        this instances's HyperSearch.

        NOTE: This is compute-intensive for HyperSearches with a huge number of
        combinations.

        NOTE/TODO:  THIS ONLY WORKS FOR RONOMATIC: This method is exposed for the
                    benefit of perutations_runner.py for use in progress
                    reporting.

        Parameters:
        ---------------------------------------------------------
        retval:       The total number of expected models, if known; -1 if unknown


return None<EOL>

Generates a list of model names that are expected to complete as part of
        this instances's HyperSearch.

        NOTE: This is compute-intensive for HyperSearches with a huge number of
        combinations.

        NOTE/TODO:  THIS ONLY WORKS FOR RONOMATIC: This method is exposed for the
                    benefit of perutations_runner.py.

        Parameters:
        ---------------------------------------------------------
        retval:       List of model names for this HypersearchV2 instance, or
                      None of not applicable


return self._flattenedPermutations<EOL>

Returns a dictionary of permutation variables.

        Parameters:
        ---------------------------------------------------------
        retval:       A dictionary of permutation variables; keys are
                      flat permutation variable names and each value is
                      a sub-class of PermuteVariable.


raise NotImplementedError<EOL>

Generates a lookup dictionary of permutation variables whose values
        are too complex for labels, so that artificial labels have to be generated
        for them.

        Parameters:
        ---------------------------------------------------------
        retval:       A look-up dictionary of permutation
                      variables whose values are too complex for labels, so
                      artificial labels were generated instead (e.g., "Choice0",
                      "Choice1", etc.); the key is the name of the complex variable
                      and the value is:
                        dict(labels=<list_of_labels>, values=<list_of_values>).


return (self._optimizeKey, self._maximize)<EOL>

Retrives the optimization key name and optimization function.

        Parameters:
        ---------------------------------------------------------
        retval:       (optimizationMetricKey, maximize)
                      optimizationMetricKey: which report key to optimize for
                      maximize: True if we should try and maximize the optimizeKey
                        metric. False if we should minimize it.


self.logger.debug("<STR_LIT>" %(self._modelOrphanIntervalSecs))<EOL>while True:<EOL><INDENT>orphanedModelId = self._cjDAO.modelAdoptNextOrphan(self._jobID,<EOL>self._modelOrphanIntervalSecs)<EOL>if orphanedModelId is None:<EOL><INDENT>return<EOL><DEDENT>self.logger.info("<STR_LIT>" % (orphanedModelId))<EOL>for attempt in range(<NUM_LIT:100>):<EOL><INDENT>paramsHash = hashlib.md5("<STR_LIT>" % (orphanedModelId,<EOL>attempt)).digest()<EOL>particleHash = hashlib.md5("<STR_LIT>" % (orphanedModelId,<EOL>attempt)).digest()<EOL>try:<EOL><INDENT>self._cjDAO.modelSetFields(orphanedModelId,<EOL>dict(engParamsHash=paramsHash,<EOL>engParticleHash=particleHash))<EOL>success = True<EOL><DEDENT>except:<EOL><INDENT>success = False<EOL><DEDENT>if success:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not success:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._cjDAO.modelSetCompleted(modelID=orphanedModelId,<EOL>completionReason=ClientJobsDAO.CMPL_REASON_ORPHAN,<EOL>completionMsg="<STR_LIT>")<EOL>self._resultsDB.update(modelID=orphanedModelId,<EOL>modelParams=None,<EOL>modelParamsHash=paramsHash,<EOL>metricResult=None,<EOL>completed = True,<EOL>completionReason = ClientJobsDAO.CMPL_REASON_ORPHAN,<EOL>matured = True,<EOL>numRecords = <NUM_LIT:0>)<EOL><DEDENT>

If there are any models that haven't been updated in a while, consider
        them dead, and mark them as hidden in our resultsDB. We also change the
        paramsHash and particleHash of orphaned models so that we can
        re-generate that particle and/or model again if we desire.

        Parameters:
        ----------------------------------------------------------------------
        retval:


if self._hsState is None:<EOL><INDENT>self._hsState =  HsState(self)<EOL><DEDENT>self._hsState.readStateFromDB()<EOL>completedSwarms = set()<EOL>if exhaustedSwarmId is not None:<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (exhaustedSwarmId))<EOL>(particles, _, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId=exhaustedSwarmId, matured=False)<EOL>if len(particles) > <NUM_LIT:0>:<EOL><INDENT>exhaustedSwarmStatus = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>exhaustedSwarmStatus = '<STR_LIT>'<EOL><DEDENT><DEDENT>if self._killUselessSwarms:<EOL><INDENT>self._hsState.killUselessSwarms()<EOL><DEDENT>completingSwarms = self._hsState.getCompletingSwarms()<EOL>for swarmId in completingSwarms:<EOL><INDENT>(particles, _, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId=swarmId, matured=False)<EOL>if len(particles) == <NUM_LIT:0>:<EOL><INDENT>completedSwarms.add(swarmId)<EOL><DEDENT><DEDENT>completedSwarmGens = self._resultsDB.getMaturedSwarmGenerations()<EOL>priorCompletedSwarms = self._hsState.getCompletedSwarms()<EOL>for (swarmId, genIdx, errScore) in completedSwarmGens:<EOL><INDENT>if swarmId in priorCompletedSwarms:<EOL><INDENT>continue<EOL><DEDENT>completedList = self._swarmTerminator.recordDataPoint(<EOL>swarmId=swarmId, generation=genIdx, errScore=errScore)<EOL>statusMsg = "<STR_LIT>""<STR_LIT>" % (genIdx, swarmId, errScore)<EOL>if len(completedList) > <NUM_LIT:0>:<EOL><INDENT>statusMsg = "<STR_LIT>" % (statusMsg, completedList)<EOL><DEDENT>self.logger.info(statusMsg)<EOL>self._cjDAO.jobSetFields (jobID=self._jobID,<EOL>fields=dict(engStatus=statusMsg),<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL>if '<STR_LIT>' in os.environ:<EOL><INDENT>while True:<EOL><INDENT>resultsStr = self._cjDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if resultsStr is None:<EOL><INDENT>results = {}<EOL><DEDENT>else:<EOL><INDENT>results = json.loads(resultsStr)<EOL><DEDENT>if not '<STR_LIT>' in results:<EOL><INDENT>results['<STR_LIT>'] = {}<EOL><DEDENT>for swarm in completedList:<EOL><INDENT>if swarm not in results['<STR_LIT>']:<EOL><INDENT>results['<STR_LIT>'][swarm] = (genIdx,<EOL>self._swarmTerminator.swarmScores[swarm])<EOL><DEDENT><DEDENT>newResultsStr = json.dumps(results)<EOL>if newResultsStr == resultsStr:<EOL><INDENT>break<EOL><DEDENT>updated = self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=resultsStr,<EOL>newValue = json.dumps(results))<EOL>if updated:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>if len(completedList) > <NUM_LIT:0>:<EOL><INDENT>for name in completedList:<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT:%s>" % (name, genIdx, errScore))<EOL><DEDENT>completedSwarms = completedSwarms.union(completedList)<EOL><DEDENT><DEDENT>if len(completedSwarms)==<NUM_LIT:0> and (exhaustedSwarmId is None):<EOL><INDENT>return<EOL><DEDENT>while True:<EOL><INDENT>if exhaustedSwarmId is not None:<EOL><INDENT>self._hsState.setSwarmState(exhaustedSwarmId, exhaustedSwarmStatus)<EOL><DEDENT>for swarmId in completedSwarms:<EOL><INDENT>self._hsState.setSwarmState(swarmId, '<STR_LIT>')<EOL><DEDENT>if not self._hsState.isDirty():<EOL><INDENT>return<EOL><DEDENT>success = self._hsState.writeStateToDB()<EOL>if success:<EOL><INDENT>jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobResultsStr is not None:<EOL><INDENT>jobResults = json.loads(jobResultsStr)<EOL>bestModelId = jobResults.get('<STR_LIT>', None)<EOL><DEDENT>else:<EOL><INDENT>bestModelId = None<EOL><DEDENT>for swarmId in list(completedSwarms):<EOL><INDENT>(_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId=swarmId, completed=False)<EOL>if bestModelId in modelIds:<EOL><INDENT>modelIds.remove(bestModelId)<EOL><DEDENT>if len(modelIds) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (swarmId,<EOL>str(modelIds)))<EOL>for modelId in modelIds:<EOL><INDENT>self._cjDAO.modelSetFields(modelId,<EOL>dict(engStop=ClientJobsDAO.STOP_REASON_KILLED),<EOL>ignoreUnchanged = True)<EOL><DEDENT><DEDENT>return<EOL><DEDENT>self._hsState.readStateFromDB()<EOL>self.logger.debug("<STR_LIT>"<EOL>"<STR_LIT>" % (pprint.pformat(self._hsState._state, indent=<NUM_LIT:4>)))<EOL><DEDENT>

Periodically, check to see if we should remove a certain field combination
from evaluation (because it is doing so poorly) or move on to the next
sprint (add in more fields).

This method is called from _getCandidateParticleAndSwarm(), which is called
right before we try and create a new model to run.

Parameters:
-----------------------------------------------------------------------
removeSwarmId:     If not None, force a change to the current set of active
                  swarms by removing this swarm. This is used in situations
                  where we can't find any new unique models to create in
                  this swarm. In these situations, we update the hypersearch
                  state regardless of the timestamp of the last time another
                  worker updated it.


def _getCandidateParticleAndSwarm (self, exhaustedSwarmId=None):

<EOL>jobCancel = self._cjDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobCancel:<EOL><INDENT>self._jobCancelled = True<EOL>(workerCmpReason, workerCmpMsg) = self._cjDAO.jobGetFields(self._jobID,<EOL>['<STR_LIT>', '<STR_LIT>'])<EOL>if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:<EOL><INDENT>self.logger.info("<STR_LIT>")<EOL>self._cjDAO.jobSetFields(self._jobID,<EOL>dict(workerCompletionMsg="<STR_LIT>"),<EOL>useConnectionID=False, ignoreUnchanged=True)<EOL><DEDENT>else:<EOL><INDENT>self.logger.error("<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(workerCmpReason, workerCmpMsg))<EOL><DEDENT>return (True, None, None)<EOL><DEDENT>if self._hsState is not None:<EOL><INDENT>priorActiveSwarms = self._hsState.getActiveSwarms()<EOL><DEDENT>else:<EOL><INDENT>priorActiveSwarms = None<EOL><DEDENT>self._hsStatePeriodicUpdate(exhaustedSwarmId=exhaustedSwarmId)<EOL>activeSwarms = self._hsState.getActiveSwarms()<EOL>if activeSwarms != priorActiveSwarms:<EOL><INDENT>self.logger.info("<STR_LIT>" % (activeSwarms,<EOL>priorActiveSwarms))<EOL><DEDENT>self.logger.debug("<STR_LIT>" % (activeSwarms))<EOL>totalCmpModels = self._resultsDB.getNumCompletedModels()<EOL>if totalCmpModels > <NUM_LIT:5>:<EOL><INDENT>numErrs = self._resultsDB.getNumErrModels()<EOL>if (float(numErrs) / totalCmpModels) > self._maxPctErrModels:<EOL><INDENT>errModelIds = self._resultsDB.getErrModelIds()<EOL>resInfo = self._cjDAO.modelsGetResultAndStatus([errModelIds[<NUM_LIT:0>]])[<NUM_LIT:0>]<EOL>modelErrMsg = resInfo.completionMsg<EOL>cmpMsg = "<STR_LIT>""<STR_LIT>" %(ErrorCodes.tooManyModelErrs, numErrs, totalCmpModels,<EOL>modelErrMsg)<EOL>self.logger.error(cmpMsg)<EOL>workerCmpReason = self._cjDAO.jobGetFields(self._jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:<EOL><INDENT>self._cjDAO.jobSetFields(<EOL>self._jobID,<EOL>fields=dict(<EOL>cancel=True,<EOL>workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,<EOL>workerCompletionMsg = cmpMsg),<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL><DEDENT>return (True, None, None)<EOL><DEDENT><DEDENT>if self._hsState.isSearchOver():<EOL><INDENT>cmpMsg = "<STR_LIT>""<STR_LIT>"<EOL>self.logger.info(cmpMsg)<EOL>self._cjDAO.jobSetFields(self._jobID,<EOL>dict(workerCompletionMsg=cmpMsg),<EOL>useConnectionID=False, ignoreUnchanged=True)<EOL>return (True, None, None)<EOL><DEDENT>sprintIdx = -<NUM_LIT:1><EOL>while True:<EOL><INDENT>sprintIdx += <NUM_LIT:1><EOL>(active, eos) = self._hsState.isSprintActive(sprintIdx)<EOL>if eos:<EOL><INDENT>if self._hsState.anyGoodSprintsActive():<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>return (False, None, None)<EOL><DEDENT>else:<EOL><INDENT>cmpMsg = "<STR_LIT>""<STR_LIT>"<EOL>self._cjDAO.jobSetFields(self._jobID,<EOL>dict(workerCompletionMsg=cmpMsg),<EOL>useConnectionID=False, ignoreUnchanged=True)<EOL>self.logger.info(cmpMsg)<EOL>return (True, None, None)<EOL><DEDENT><DEDENT>if not active:<EOL><INDENT>if not self._speculativeParticles:<EOL><INDENT>if not self._hsState.isSprintCompleted(sprintIdx):<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (sprintIdx))<EOL>return (False, None, None)<EOL><DEDENT><DEDENT>continue<EOL><DEDENT>swarmIds = self._hsState.getActiveSwarms(sprintIdx)<EOL>for swarmId in swarmIds:<EOL><INDENT>firstNonFullGenIdx = self._resultsDB.firstNonFullGeneration(<EOL>swarmId=swarmId,<EOL>minNumParticles=self._minParticlesPerSwarm)<EOL>if firstNonFullGenIdx is None:<EOL><INDENT>continue<EOL><DEDENT>if firstNonFullGenIdx < self._resultsDB.highestGeneration(swarmId):<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>firstNonFullGenIdx, swarmId, sprintIdx))<EOL>(allParticles, allModelIds, errScores, completed, matured) =self._resultsDB.getOrphanParticleInfos(swarmId, firstNonFullGenIdx)<EOL>if len(allModelIds) > <NUM_LIT:0>:<EOL><INDENT>newParticleId = True<EOL>self.logger.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>newParticleId = True<EOL>self.logger.info("<STR_LIT>")<EOL>(allParticles, allModelIds, errScores, completed, matured) =self._resultsDB.getParticleInfos(swarmId=swarmId,<EOL>genIdx=firstNonFullGenIdx)<EOL><DEDENT>modelId = random.choice(allModelIds)<EOL>self.logger.info("<STR_LIT>" % (modelId))<EOL>(particleState, _, _, _, _) = self._resultsDB.getParticleInfo(modelId)<EOL>particle = Particle(hsObj = self,<EOL>resultsDB = self._resultsDB,<EOL>flattenedPermuteVars=self._flattenedPermutations,<EOL>newFromClone=particleState,<EOL>newParticleId=newParticleId)<EOL>return (False, particle, swarmId)<EOL><DEDENT><DEDENT>swarmSizes = numpy.array([self._resultsDB.numModels(x) for x in swarmIds])<EOL>swarmSizeAndIdList = list(zip(swarmSizes, swarmIds))<EOL>swarmSizeAndIdList.sort()<EOL>for (_, swarmId) in swarmSizeAndIdList:<EOL><INDENT>(allParticles, allModelIds, errScores, completed, matured) = (<EOL>self._resultsDB.getParticleInfos(swarmId))<EOL>if len(allParticles) < self._minParticlesPerSwarm:<EOL><INDENT>particle = Particle(hsObj=self,<EOL>resultsDB=self._resultsDB,<EOL>flattenedPermuteVars=self._flattenedPermutations,<EOL>swarmId=swarmId,<EOL>newFarFrom=allParticles)<EOL>bestPriorModel = None<EOL>if sprintIdx >= <NUM_LIT:1>:<EOL><INDENT>(bestPriorModel, errScore) = self._hsState.bestModelInSprint(<NUM_LIT:0>)<EOL><DEDENT>if bestPriorModel is not None:<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (<NUM_LIT:0>, str(bestPriorModel), errScore))<EOL>(baseState, modelId, errScore, completed, matured)= self._resultsDB.getParticleInfo(bestPriorModel)<EOL>particle.copyEncoderStatesFrom(baseState)<EOL>particle.copyVarStatesFrom(baseState, ['<STR_LIT>'])<EOL>whichVars = []<EOL>for varName in baseState['<STR_LIT>']:<EOL><INDENT>if '<STR_LIT::>' in varName:<EOL><INDENT>whichVars.append(varName)<EOL><DEDENT><DEDENT>particle.newPosition(whichVars)<EOL>self.logger.debug("<STR_LIT>"<EOL>"<STR_LIT>" % (str(particle)))<EOL><DEDENT>return (False, particle, swarmId)<EOL><DEDENT>(readyParticles, readyModelIds, readyErrScores, _, _) = (<EOL>self._resultsDB.getParticleInfos(swarmId, genIdx=None,<EOL>matured=True, lastDescendent=True))<EOL>if len(readyParticles) > <NUM_LIT:0>:<EOL><INDENT>readyGenIdxs = [x['<STR_LIT>'] for x in readyParticles]<EOL>sortedGenIdxs = sorted(set(readyGenIdxs))<EOL>genIdx = sortedGenIdxs[<NUM_LIT:0>]<EOL>useParticle = None<EOL>for particle in readyParticles:<EOL><INDENT>if particle['<STR_LIT>'] == genIdx:<EOL><INDENT>useParticle = particle<EOL>break<EOL><DEDENT><DEDENT>if not self._speculativeParticles:<EOL><INDENT>(particles, _, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId, genIdx=genIdx, matured=False)<EOL>if len(particles) > <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>particle = Particle(hsObj=self,<EOL>resultsDB=self._resultsDB,<EOL>flattenedPermuteVars=self._flattenedPermutations,<EOL>evolveFromState=useParticle)<EOL>return (False, particle, swarmId)<EOL><DEDENT><DEDENT>if not self._speculativeParticles:<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>"% (str(swarmIds)))<EOL>return (False, None, None)<EOL><DEDENT><DEDENT>

Find or create a candidate particle to produce a new model.

        At any one time, there is an active set of swarms in the current sprint, where
        each swarm in the sprint represents a particular combination of fields.
        Ideally, we should try to balance the number of models we have evaluated for
        each swarm at any time.

        This method will see how many models have been evaluated for each active
        swarm in the current active sprint(s) and then try and choose a particle
        from the least represented swarm in the first possible active sprint, with
        the following constraints/rules:

        for each active sprint:
          for each active swarm (preference to those with least# of models so far):
            1.) The particle will be created from new (generation #0) if there are not
            already self._minParticlesPerSwarm particles in the swarm.

            2.) Find the first gen that has a completed particle and evolve that
            particle to the next generation.

            3.) If we got to here, we know that we have satisfied the min# of
            particles for the swarm, and they are all currently running (probably at
            various generation indexes). Go onto the next swarm

          If we couldn't find a swarm to allocate a particle in, go onto the next
          sprint and start allocating particles there....

        Parameters:
        ----------------------------------------------------------------
        exhaustedSwarmId:   If not None, force a change to the current set of active
                            swarms by marking this swarm as either 'completing' or
                            'completed'. If there are still models being evaluaed in
                            it, mark it as 'completing', else 'completed. This is
                            used in situations where we can't find any new unique
                            models to create in this swarm. In these situations, we
                            force an update to the hypersearch state so no other
                            worker wastes time try to use this swarm.

        retval: (exit, particle, swarm)
                  exit: If true, this worker is ready to exit (particle and
                          swarm will be None)
                  particle: Which particle to run
                  swarm: which swarm the particle is in

                  NOTE: When particle and swarm are None and exit is False, it
                  means that we need to wait for one or more other worker(s) to
                  finish their respective models before we can pick a particle
                  to run. This will generally only happen when speculativeParticles
                  is set to False.


<EOL>print("<STR_LIT>", file=sys.stderr)<EOL>if not self._jobCancelled:<EOL><INDENT>(_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(matured=False)<EOL>if len(modelIds) > <NUM_LIT:0>:<EOL><INDENT>self.logger.info("<STR_LIT>""<STR_LIT>""<STR_LIT>")<EOL>time.sleep(<NUM_LIT> * random.random())<EOL>return False<EOL><DEDENT><DEDENT>(_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(completed=False)<EOL>for modelId in modelIds:<EOL><INDENT>self.logger.info("<STR_LIT>"% (modelId))<EOL>self._cjDAO.modelSetFields(modelId,<EOL>dict(engStop=ClientJobsDAO.STOP_REASON_STOPPED),<EOL>ignoreUnchanged = True)<EOL><DEDENT>self._hsStatePeriodicUpdate()<EOL>pctFieldContributions, absFieldContributions =self._hsState.getFieldContributions()<EOL>jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobResultsStr is not None:<EOL><INDENT>jobResults = json.loads(jobResultsStr)<EOL><DEDENT>else:<EOL><INDENT>jobResults = {}<EOL><DEDENT>if pctFieldContributions != jobResults.get('<STR_LIT>', None):<EOL><INDENT>jobResults['<STR_LIT>'] = pctFieldContributions<EOL>jobResults['<STR_LIT>'] = absFieldContributions<EOL>isUpdated = self._cjDAO.jobSetFieldIfEqual(self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=jobResultsStr,<EOL>newValue=json.dumps(jobResults))<EOL>if isUpdated:<EOL><INDENT>self.logger.info('<STR_LIT>',<EOL>pctFieldContributions)<EOL><DEDENT>else:<EOL><INDENT>self.logger.info('<STR_LIT>''<STR_LIT>')<EOL><DEDENT><DEDENT>return True<EOL>

Test if it's OK to exit this worker. This is only called when we run
        out of prospective new models to evaluate. This method sees if all models
        have matured yet. If not, it will sleep for a bit and return False. This
        will indicate to the hypersearch worker that we should keep running, and
        check again later. This gives this worker a chance to pick up and adopt any
        model which may become orphaned by another worker before it matures.

        If all models have matured, this method will send a STOP message to all
        matured, running models (presummably, there will be just one - the model
        which thinks it's the best) before returning True.


<EOL>self._checkForOrphanedModels()<EOL>modelResults = []<EOL>for _ in range(numModels):<EOL><INDENT>candidateParticle = None<EOL>if (self._maxModels is not None and<EOL>(self._resultsDB.numModels() - self._resultsDB.getNumErrModels()) >=<EOL>self._maxModels):<EOL><INDENT>return (self._okToExit(), [])<EOL><DEDENT>if candidateParticle is None:<EOL><INDENT>(exitNow, candidateParticle, candidateSwarm) = (<EOL>self._getCandidateParticleAndSwarm())<EOL><DEDENT>if candidateParticle is None:<EOL><INDENT>if exitNow:<EOL><INDENT>return (self._okToExit(), [])<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>", file=sys.stderr)<EOL>time.sleep(self._speculativeWaitSecondsMax * random.random())<EOL>return (False, [])<EOL><DEDENT><DEDENT>useEncoders = candidateSwarm.split('<STR_LIT:.>')<EOL>numAttempts = <NUM_LIT:0><EOL>while True:<EOL><INDENT>if numAttempts >= <NUM_LIT:1>:<EOL><INDENT>self.logger.debug("<STR_LIT>"<EOL>"<STR_LIT>" % (numAttempts))<EOL>candidateParticle.agitate()<EOL><DEDENT>position = candidateParticle.getPosition()<EOL>structuredParams = dict()<EOL>def _buildStructuredParams(value, keys):<EOL><INDENT>flatKey = _flattenKeys(keys)<EOL>if flatKey in self._encoderNames:<EOL><INDENT>if flatKey in useEncoders:<EOL><INDENT>return value.getDict(flatKey, position)<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>elif flatKey in position:<EOL><INDENT>return position[flatKey]<EOL><DEDENT>else:<EOL><INDENT>return value<EOL><DEDENT><DEDENT>structuredParams = rCopy(self._permutations,<EOL>_buildStructuredParams,<EOL>discardNoneKeys=False)<EOL>modelParams = dict(<EOL>structuredParams=structuredParams,<EOL>particleState = candidateParticle.getState()<EOL>)<EOL>m = hashlib.md5()<EOL>m.update(sortedJSONDumpS(structuredParams))<EOL>m.update(self._baseDescriptionHash)<EOL>paramsHash = m.digest()<EOL>particleInst = "<STR_LIT>" % (modelParams['<STR_LIT>']['<STR_LIT:id>'],<EOL>modelParams['<STR_LIT>']['<STR_LIT>'])<EOL>particleHash = hashlib.md5(particleInst).digest()<EOL>numAttempts += <NUM_LIT:1><EOL>if self._filterFunc and not self._filterFunc(structuredParams):<EOL><INDENT>valid = False<EOL><DEDENT>else:<EOL><INDENT>valid = True<EOL><DEDENT>if valid and self._resultsDB.getModelIDFromParamsHash(paramsHash) is None:<EOL><INDENT>break<EOL><DEDENT>if numAttempts >= self._maxUniqueModelAttempts:<EOL><INDENT>(exitNow, candidateParticle, candidateSwarm)= self._getCandidateParticleAndSwarm(<EOL>exhaustedSwarmId=candidateSwarm)<EOL>if candidateParticle is None:<EOL><INDENT>if exitNow:<EOL><INDENT>return (self._okToExit(), [])<EOL><DEDENT>else:<EOL><INDENT>time.sleep(self._speculativeWaitSecondsMax * random.random())<EOL>return (False, [])<EOL><DEDENT><DEDENT>numAttempts = <NUM_LIT:0><EOL>useEncoders = candidateSwarm.split('<STR_LIT:.>')<EOL><DEDENT><DEDENT>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>self.logger.debug("<STR_LIT>"<EOL>% (pprint.pformat(modelParams, indent=<NUM_LIT:4>)))<EOL><DEDENT>modelResults.append((modelParams, paramsHash, particleHash))<EOL><DEDENT>return (False, modelResults)<EOL>

Create one or more new models for evaluation. These should NOT be models
        that we already know are in progress (i.e. those that have been sent to us
        via recordModelProgress). We return a list of models to the caller
        (HypersearchWorker) and if one can be successfully inserted into
        the models table (i.e. it is not a duplicate) then HypersearchWorker will
        turn around and call our runModel() method, passing in this model. If it
        is a duplicate, HypersearchWorker will call this method again. A model
        is a duplicate if either the  modelParamsHash or particleHash is
        identical to another entry in the model table.

        The numModels is provided by HypersearchWorker as a suggestion as to how
        many models to generate. This particular implementation only ever returns 1
        model.

        Before choosing some new models, we first do a sweep for any models that
        may have been abandonded by failed workers. If/when we detect an abandoned
        model, we mark it as complete and orphaned and hide it from any subsequent
        queries to our ResultsDB. This effectively considers it as if it never
        existed. We also change the paramsHash and particleHash in the model record
        of the models table so that we can create another model with the same
        params and particle status and run it (which we then do immediately).

        The modelParamsHash returned for each model should be a hash (max allowed
        size of ClientJobsDAO.hashMaxSize) that uniquely identifies this model by
        it's params and the optional particleHash should be a hash of the particleId
        and generation index. Every model that gets placed into the models database,
        either by this worker or another worker, will have these hashes computed for
        it. The recordModelProgress gets called for every model in the database and
        the hash is used to tell which, if any, are the same as the ones this worker
        generated.

        NOTE: We check first ourselves for possible duplicates using the paramsHash
        before we return a model. If HypersearchWorker failed to insert it (because
        some other worker beat us to it), it will turn around and call our
        recordModelProgress with that other model so that we now know about it. It
        will then call createModels() again.

        This methods returns an exit boolean and the model to evaluate. If there is
        no model to evalulate, we may return False for exit because we want to stay
        alive for a while, waiting for all other models to finish. This gives us
        a chance to detect and pick up any possibly orphaned model by another
        worker.

        Parameters:
        ----------------------------------------------------------------------
        numModels:   number of models to generate
        retval:      (exit, models)
                        exit: true if this worker should exit.
                        models: list of tuples, one for each model. Each tuple contains:
                          (modelParams, modelParamsHash, particleHash)

                     modelParams is a dictionary containing the following elements:

                       structuredParams: dictionary containing all variables for
                         this model, with encoders represented as a dict within
                         this dict (or None if they are not included.

                       particleState: dictionary containing the state of this
                         particle. This includes the position and velocity of
                         each of it's variables, the particleId, and the particle
                         generation index. It contains the following keys:

                         id: The particle Id of the particle we are using to
                               generate/track this model. This is a string of the
                               form <hypesearchWorkerId>.<particleIdx>
                         genIdx: the particle's generation index. This starts at 0
                               and increments every time we move the particle to a
                               new position.
                         swarmId: The swarmId, which is a string of the form
                           <encoder>.<encoder>... that describes this swarm
                         varStates: dict of the variable states. The key is the
                             variable name, the value is a dict of the variable's
                             position, velocity, bestPosition, bestResult, etc.


def recordModelProgress(self, modelID, modelParams, modelParamsHash, results,<EOL>completed, completionReason, matured, numRecords):

if results is None:<EOL><INDENT>metricResult = None<EOL><DEDENT>else:<EOL><INDENT>metricResult = list(results[<NUM_LIT:1>].values())[<NUM_LIT:0>]<EOL><DEDENT>errScore = self._resultsDB.update(modelID=modelID,<EOL>modelParams=modelParams,modelParamsHash=modelParamsHash,<EOL>metricResult=metricResult, completed=completed,<EOL>completionReason=completionReason, matured=matured,<EOL>numRecords=numRecords)<EOL>self.logger.debug('<STR_LIT>'<EOL>'<STR_LIT>' ,<EOL>modelID, completed, completionReason, numRecords, errScore)<EOL>(bestModelID, bestResult) = self._resultsDB.bestModelIdAndErrScore()<EOL>self.logger.debug('<STR_LIT>' %(bestResult, bestModelID))<EOL>

Record or update the results for a model. This is called by the
        HSW whenever it gets results info for another model, or updated results
        on a model that is still running.

        The first time this is called for a given modelID, the modelParams will
        contain the params dict for that model and the modelParamsHash will contain
        the hash of the params. Subsequent updates of the same modelID will
        have params and paramsHash values of None (in order to save overhead).

        The Hypersearch object should save these results into it's own working
        memory into some table, which it then uses to determine what kind of
        new models to create next time createModels() is called.

        Parameters:
        ----------------------------------------------------------------------
        modelID:        ID of this model in models table
        modelParams:    params dict for this model, or None if this is just an update
                        of a model that it already previously reported on.

                        See the comments for the createModels() method for a
                        description of this dict.

        modelParamsHash:  hash of the modelParams dict, generated by the worker
                        that put it into the model database.
        results:        tuple containing (allMetrics, optimizeMetric). Each is a
                        dict containing metricName:result pairs. .
                        May be none if we have no results yet.
        completed:      True if the model has completed evaluation, False if it
                          is still running (and these are online results)
        completionReason: One of the ClientJobsDAO.CMPL_REASON_XXX equates
        matured:        True if this model has matured. In most cases, once a
                        model matures, it will complete as well. The only time a
                        model matures and does not complete is if it's currently
                        the best model and we choose to keep it running to generate
                        predictions.
        numRecords:     Number of records that have been processed so far by this
                          model.


def runModel(self, modelID, jobID, modelParams, modelParamsHash,<EOL>jobsDAO, modelCheckpointGUID):

<EOL>if not self._createCheckpoints:<EOL><INDENT>modelCheckpointGUID = None<EOL><DEDENT>self._resultsDB.update(modelID=modelID,<EOL>modelParams=modelParams,<EOL>modelParamsHash=modelParamsHash,<EOL>metricResult = None,<EOL>completed = False,<EOL>completionReason = None,<EOL>matured = False,<EOL>numRecords = <NUM_LIT:0>)<EOL>structuredParams = modelParams['<STR_LIT>']<EOL>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>self.logger.debug("<STR_LIT>" %(pprint.pformat(modelParams, indent=<NUM_LIT:4>), modelID))<EOL><DEDENT>cpuTimeStart = time.clock()<EOL>logLevel = self.logger.getEffectiveLevel()<EOL>try:<EOL><INDENT>if self._dummyModel is None or self._dummyModel is False:<EOL><INDENT>(cmpReason, cmpMsg) = runModelGivenBaseAndParams(<EOL>modelID=modelID,<EOL>jobID=jobID,<EOL>baseDescription=self._baseDescription,<EOL>params=structuredParams,<EOL>predictedField=self._predictedField,<EOL>reportKeys=self._reportKeys,<EOL>optimizeKey=self._optimizeKey,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=self._predictionCacheMaxRecords)<EOL><DEDENT>else:<EOL><INDENT>dummyParams = dict(self._dummyModel)<EOL>dummyParams['<STR_LIT>'] = structuredParams<EOL>if self._dummyModelParamsFunc is not None:<EOL><INDENT>permInfo = dict(structuredParams)<EOL>permInfo ['<STR_LIT>'] = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>dummyParams.update(self._dummyModelParamsFunc(permInfo))<EOL><DEDENT>(cmpReason, cmpMsg) = runDummyModel(<EOL>modelID=modelID,<EOL>jobID=jobID,<EOL>params=dummyParams,<EOL>predictedField=self._predictedField,<EOL>reportKeys=self._reportKeys,<EOL>optimizeKey=self._optimizeKey,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=self._predictionCacheMaxRecords)<EOL><DEDENT>jobsDAO.modelSetCompleted(modelID,<EOL>completionReason = cmpReason,<EOL>completionMsg = cmpMsg,<EOL>cpuTime = time.clock() - cpuTimeStart)<EOL><DEDENT>except InvalidConnectionException as e:<EOL><INDENT>self.logger.warn("<STR_LIT:%s>", e)<EOL><DEDENT>

Run the given model.

        This runs the model described by 'modelParams'. Periodically, it updates
        the results seen on the model to the model database using the databaseAO
        (database Access Object) methods.

        Parameters:
        -------------------------------------------------------------------------
        modelID:             ID of this model in models table

        jobID:               ID for this hypersearch job in the jobs table

        modelParams:         parameters of this specific model
                             modelParams is a dictionary containing the name/value
                             pairs of each variable we are permuting over. Note that
                             variables within an encoder spec have their name
                             structure as:
                               <encoderName>.<encodrVarName>

        modelParamsHash:     hash of modelParamValues

        jobsDAO              jobs data access object - the interface to the jobs
                              database where model information is stored

        modelCheckpointGUID: A persistent, globally-unique identifier for
                              constructing the model checkpoint key


raise NotImplementedError<EOL>

Return the current state of this particle. This is used for
        communicating our state into a model record entry so that it can be
        instantiated on another worker.


raise NotImplementedError<EOL>

Set the current state of this particle. This is counterpart to getState.


raise NotImplementedError<EOL>

for int vars, returns position to nearest int

        Parameters:
        --------------------------------------------------------------
        retval:     current position


raise NotImplementedError<EOL>

This causes the variable to jiggle away from its current position.
        It does this by increasing its velocity by a multiplicative factor.
        Every time agitate() is called, the velocity will increase. In this way,
        you can call agitate over and over again until the variable reaches a
        new position.


def newPosition(self, globalBestPosition, rng):

raise NotImplementedError<EOL>

Choose a new position based on results obtained so far from other
        particles and the passed in globalBestPosition.

        Parameters:
        --------------------------------------------------------------
        globalBestPosition:   global best position for this colony
        rng:                  instance of random.Random() used for generating
                                  random numbers
        retval:       new position


raise NotImplementedError<EOL>

Choose a new position that is as far away as possible from all
        'otherVars', where 'otherVars' is a list of PermuteVariable instances.

        Parameters:
        --------------------------------------------------------------
        otherVars:   list of other PermuteVariables to push away from
        rng:                  instance of random.Random() used for generating
                                  random numbers


raise NotImplementedError<EOL>

Reset the velocity to be some fraction of the total distance. This
        is called usually when we start a new swarm and want to start at the
        previous best position found in the previous swarm but with a
        velocity which is a known fraction of the total distance between min
        and max.

        Parameters:
        --------------------------------------------------------------
        rng:                  instance of random.Random() used for generating
                                  random numbers


def __init__(self, min, max, stepSize=None, inertia=None, cogRate=None,<EOL>socRate=None):

super(PermuteFloat, self).__init__()<EOL>self.min = min<EOL>self.max = max<EOL>self.stepSize = stepSize<EOL>self._position = (self.max + self.min) / <NUM_LIT><EOL>self._velocity = (self.max - self.min) / <NUM_LIT><EOL>self._inertia = (float(Configuration.get("<STR_LIT>"))<EOL>if inertia is None else inertia)<EOL>self._cogRate = (float(Configuration.get("<STR_LIT>"))<EOL>if cogRate is None else cogRate)<EOL>self._socRate = (float(Configuration.get("<STR_LIT>"))<EOL>if socRate is None else socRate)<EOL>self._bestPosition = self.getPosition()<EOL>self._bestResult = None<EOL>

Construct a variable that permutes over floating point values using
        the Particle Swarm Optimization (PSO) algorithm. See descriptions of
        PSO (i.e. http://en.wikipedia.org/wiki/Particle_swarm_optimization)
        for references to the inertia, cogRate, and socRate parameters.

        Parameters:
        -----------------------------------------------------------------------
        min:          min allowed value of position
        max:          max allowed value of position
        stepSize:     if not None, the position must be at min + N * stepSize,
                        where N is an integer
        inertia:      The inertia for the particle.
        cogRate:      This parameter controls how much the particle is affected
                        by its distance from it's local best position
        socRate:      This parameter controls how much the particle is affected
                        by its distance from the global best position


return ("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>self.min, self.max, self.stepSize, self.getPosition(),<EOL>self._position, self._velocity, self._bestPosition,<EOL>self._bestResult))<EOL>


return dict(_position = self._position,<EOL>position = self.getPosition(),<EOL>velocity = self._velocity,<EOL>bestPosition = self._bestPosition,<EOL>bestResult = self._bestResult)<EOL>


self._position = state['<STR_LIT>']<EOL>self._velocity = state['<STR_LIT>']<EOL>self._bestPosition = state['<STR_LIT>']<EOL>self._bestResult = state['<STR_LIT>']<EOL>


if self.stepSize is None:<EOL><INDENT>return self._position<EOL><DEDENT>numSteps = (self._position - self.min)  / self.stepSize<EOL>numSteps = int(round(numSteps))<EOL>position = self.min + (numSteps * self.stepSize)<EOL>position = max(self.min, position)<EOL>position = min(self.max, position)<EOL>return position<EOL>


<EOL>self._velocity *= <NUM_LIT> / self._inertia<EOL>maxV = (self.max - self.min)/<NUM_LIT:2><EOL>if self._velocity > maxV:<EOL><INDENT>self._velocity = maxV<EOL><DEDENT>elif self._velocity < -maxV:<EOL><INDENT>self._velocity = -maxV<EOL><DEDENT>if self._position == self.max and self._velocity > <NUM_LIT:0>:<EOL><INDENT>self._velocity *= -<NUM_LIT:1><EOL><DEDENT>if self._position == self.min and self._velocity < <NUM_LIT:0>:<EOL><INDENT>self._velocity *= -<NUM_LIT:1><EOL><DEDENT>


<EOL>lb=float(Configuration.get("<STR_LIT>"))<EOL>ub=float(Configuration.get("<STR_LIT>"))<EOL>self._velocity = (self._velocity * self._inertia + rng.uniform(lb, ub) *<EOL>self._cogRate * (self._bestPosition - self.getPosition()))<EOL>if globalBestPosition is not None:<EOL><INDENT>self._velocity += rng.uniform(lb, ub) * self._socRate * (<EOL>globalBestPosition - self.getPosition())<EOL><DEDENT>self._position += self._velocity<EOL>self._position = max(self.min, self._position)<EOL>self._position = min(self.max, self._position)<EOL>return self.getPosition()<EOL>


def pushAwayFrom(self, otherPositions, rng):

<EOL>if self.max == self.min:<EOL><INDENT>return<EOL><DEDENT>numPositions = len(otherPositions) * <NUM_LIT:4><EOL>if numPositions == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>stepSize = float(self.max-self.min) / numPositions<EOL>positions = numpy.arange(self.min, self.max + stepSize, stepSize)<EOL>numPositions = len(positions)<EOL>weights = numpy.zeros(numPositions)<EOL>maxDistanceSq = -<NUM_LIT:1> * (stepSize ** <NUM_LIT:2>)<EOL>for pos in otherPositions:<EOL><INDENT>distances = pos - positions<EOL>varWeights = numpy.exp(numpy.power(distances, <NUM_LIT:2>) / maxDistanceSq)<EOL>weights += varWeights<EOL><DEDENT>positionIdx = weights.argmin()<EOL>self._position = positions[positionIdx]<EOL>self._bestPosition = self.getPosition()<EOL>self._velocity *= rng.choice([<NUM_LIT:1>, -<NUM_LIT:1>])<EOL>


maxVelocity = (self.max - self.min) / <NUM_LIT><EOL>self._velocity = maxVelocity <EOL>self._velocity *= rng.choice([<NUM_LIT:1>, -<NUM_LIT:1>])<EOL>


position = super(PermuteInt, self).getPosition()<EOL>position =  int(round(position))<EOL>return position<EOL>


return "<STR_LIT>" % (self.choices,<EOL>self.choices[self._positionIdx])<EOL>


return dict(_position = self.getPosition(),<EOL>position = self.getPosition(),<EOL>velocity = None,<EOL>bestPosition = self.choices[self._bestPositionIdx],<EOL>bestResult = self._bestResult)<EOL>


self._positionIdx = self.choices.index(state['<STR_LIT>'])<EOL>self._bestPositionIdx = self.choices.index(state['<STR_LIT>'])<EOL>self._bestResult = state['<STR_LIT>']<EOL>


<EOL>self._resultsPerChoice = [[]] * len(self.choices)<EOL>for (choiceValue, values) in resultsPerChoice:<EOL><INDENT>choiceIndex = self.choices.index(choiceValue)<EOL>self._resultsPerChoice[choiceIndex] = list(values)<EOL><DEDENT>

Setup our resultsPerChoice history based on the passed in
        resultsPerChoice.

        For example, if this variable has the following choices:
          ['a', 'b', 'c']

        resultsPerChoice will have up to 3 elements, each element is a tuple
        containing (choiceValue, errors) where errors is the list of errors
        received from models that used the specific choice:
        retval:
          [('a', [0.1, 0.2, 0.3]), ('b', [0.5, 0.1, 0.6]), ('c', [0.2])]


return self.choices[self._positionIdx]<EOL>


<EOL>pass<EOL>


<EOL>numChoices = len(self.choices)<EOL>meanScorePerChoice = []<EOL>overallSum = <NUM_LIT:0><EOL>numResults = <NUM_LIT:0><EOL>for i in range(numChoices):<EOL><INDENT>if len(self._resultsPerChoice[i]) > <NUM_LIT:0>:<EOL><INDENT>data = numpy.array(self._resultsPerChoice[i])<EOL>meanScorePerChoice.append(data.mean())<EOL>overallSum += data.sum()<EOL>numResults += data.size<EOL><DEDENT>else:<EOL><INDENT>meanScorePerChoice.append(None)<EOL><DEDENT><DEDENT>if numResults == <NUM_LIT:0>:<EOL><INDENT>overallSum = <NUM_LIT:1.0><EOL>numResults = <NUM_LIT:1><EOL><DEDENT>for i in range(numChoices):<EOL><INDENT>if meanScorePerChoice[i] is None:<EOL><INDENT>meanScorePerChoice[i] = overallSum / numResults<EOL><DEDENT><DEDENT>meanScorePerChoice = numpy.array(meanScorePerChoice)<EOL>meanScorePerChoice = (<NUM_LIT> * meanScorePerChoice.max()) - meanScorePerChoice<EOL>if self._fixEarly:<EOL><INDENT>meanScorePerChoice **= (numResults * self._fixEarlyFactor / numChoices)<EOL><DEDENT>total = meanScorePerChoice.sum()<EOL>if total == <NUM_LIT:0>:<EOL><INDENT>total = <NUM_LIT:1.0><EOL><DEDENT>meanScorePerChoice /= total<EOL>distribution = meanScorePerChoice.cumsum()<EOL>r = rng.random() * distribution[-<NUM_LIT:1>]<EOL>choiceIdx = numpy.where(r <= distribution)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>self._positionIdx = choiceIdx<EOL>return self.getPosition()<EOL>


<EOL>positions = [self.choices.index(x) for x in otherPositions]<EOL>positionCounts = [<NUM_LIT:0>] * len(self.choices)<EOL>for pos in positions:<EOL><INDENT>positionCounts[pos] += <NUM_LIT:1><EOL><DEDENT>self._positionIdx = numpy.array(positionCounts).argmin()<EOL>self._bestPositionIdx = self._positionIdx<EOL>


pass<EOL>


suffix = "<STR_LIT>"<EOL>for key, value in list(self.kwArgs.items()):<EOL><INDENT>suffix += "<STR_LIT>" % (key, value)<EOL><DEDENT>return "<STR_LIT>" % (<EOL>(self.fieldName, self.encoderClass, self.name, suffix))<EOL>


def getDict(self, encoderName, flattenedChosenValues):

encoder = dict(fieldname=self.fieldName,<EOL>name=self.name)<EOL>for encoderArg, value in self.kwArgs.items():<EOL><INDENT>if isinstance(value, PermuteVariable):<EOL><INDENT>value = flattenedChosenValues["<STR_LIT>" % (encoderName, encoderArg)]<EOL><DEDENT>encoder[encoderArg] = value<EOL><DEDENT>if '<STR_LIT:.>' in self.encoderClass:<EOL><INDENT>(encoder['<STR_LIT:type>'], argName) = self.encoderClass.split('<STR_LIT:.>')<EOL>argValue = (encoder['<STR_LIT:w>'], encoder['<STR_LIT>'])<EOL>encoder[argName] = argValue<EOL>encoder.pop('<STR_LIT:w>')<EOL>encoder.pop('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>encoder['<STR_LIT:type>'] = self.encoderClass<EOL><DEDENT>return encoder<EOL>

Return a dict that can be used to construct this encoder. This dict
        can be passed directly to the addMultipleEncoders() method of the
        multi encoder.

        Parameters:
        ----------------------------------------------------------------------
        encoderName:            name of the encoder
        flattenedChosenValues:  dict of the flattened permutation variables. Any
                                  variables within this dict whose key starts
                                  with encoderName will be substituted for
                                  encoder constructor args which are being
                                  permuted over.


<EOL>self.verbosity = <NUM_LIT:0><EOL>self._testValidPositions(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, stepSize=<NUM_LIT:0.5>)<EOL>self._testValidPositions(varClass=PermuteInt, minValue=<NUM_LIT:2>,<EOL>maxValue=<NUM_LIT:11>, stepSize=<NUM_LIT:3>)<EOL>self._testValidPositions(varClass=PermuteInt, minValue=<NUM_LIT:2>,<EOL>maxValue=<NUM_LIT:11>, stepSize=<NUM_LIT:1>)<EOL>self._testConvergence(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteInt, minValue=<NUM_LIT:1>,<EOL>maxValue=<NUM_LIT:20>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteInt, minValue=<NUM_LIT:1>,<EOL>maxValue=<NUM_LIT:20>, targetValue=<NUM_LIT:1>)<EOL>self._testChoices()<EOL>


lines = [l.rstrip() for l in s.split('<STR_LIT:\n>')]<EOL>return '<STR_LIT:\n>'.join(lines)<EOL>

Removes trailing whitespace on each line.


def __init__(self, hsObj, resultsDB, flattenedPermuteVars,<EOL>swarmId=None, newFarFrom=None, evolveFromState=None,<EOL>newFromClone=None, newParticleId=False):

<EOL>self._hsObj = hsObj<EOL>self.logger = hsObj.logger<EOL>self._resultsDB = resultsDB<EOL>self._rng = random.Random()<EOL>self._rng.seed(<NUM_LIT>)<EOL>def _setupVars(flattenedPermuteVars):<EOL><INDENT>allowedEncoderNames = self.swarmId.split('<STR_LIT:.>')<EOL>self.permuteVars = copy.deepcopy(flattenedPermuteVars)<EOL>varNames = self.permuteVars.keys()<EOL>for varName in varNames:<EOL><INDENT>if '<STR_LIT::>' in varName:  <EOL><INDENT>if varName.split('<STR_LIT::>')[<NUM_LIT:0>] not in allowedEncoderNames:<EOL><INDENT>self.permuteVars.pop(varName)<EOL>continue<EOL><DEDENT><DEDENT>if isinstance(self.permuteVars[varName], PermuteChoices):<EOL><INDENT>if self._hsObj._speculativeParticles:<EOL><INDENT>maxGenIdx = None<EOL><DEDENT>else:<EOL><INDENT>maxGenIdx = self.genIdx - <NUM_LIT:1><EOL><DEDENT>resultsPerChoice = self._resultsDB.getResultsPerChoice(<EOL>swarmId=self.swarmId, maxGenIdx=maxGenIdx, varName=varName)<EOL>self.permuteVars[varName].setResultsPerChoice(<EOL>resultsPerChoice.values())<EOL><DEDENT><DEDENT><DEDENT>if swarmId is not None:<EOL><INDENT>assert (evolveFromState is None)<EOL>assert (newFromClone is None)<EOL>self.swarmId = swarmId<EOL>self.particleId = "<STR_LIT>" % (str(self._hsObj._workerID),<EOL>str(Particle._nextParticleID))<EOL>Particle._nextParticleID += <NUM_LIT:1><EOL>self.genIdx = <NUM_LIT:0><EOL>_setupVars(flattenedPermuteVars)<EOL>if newFarFrom is not None:<EOL><INDENT>for varName in self.permuteVars.iterkeys():<EOL><INDENT>otherPositions = []<EOL>for particleState in newFarFrom:<EOL><INDENT>otherPositions.append(<EOL>particleState['<STR_LIT>'][varName]['<STR_LIT>'])<EOL><DEDENT>self.permuteVars[varName].pushAwayFrom(otherPositions, self._rng)<EOL>self._rng.seed(str(otherPositions))<EOL><DEDENT><DEDENT><DEDENT>elif evolveFromState is not None:<EOL><INDENT>assert (swarmId is None)<EOL>assert (newFarFrom is None)<EOL>assert (newFromClone is None)<EOL>self.particleId = evolveFromState['<STR_LIT:id>']<EOL>self.genIdx = evolveFromState['<STR_LIT>'] + <NUM_LIT:1><EOL>self.swarmId = evolveFromState['<STR_LIT>']<EOL>_setupVars(flattenedPermuteVars)<EOL>self.initStateFrom(self.particleId, evolveFromState, newBest=True)<EOL>self.newPosition()<EOL><DEDENT>elif newFromClone is not None:<EOL><INDENT>assert (swarmId is None)<EOL>assert (newFarFrom is None)<EOL>assert (evolveFromState is None)<EOL>self.particleId = newFromClone['<STR_LIT:id>']<EOL>if newParticleId:<EOL><INDENT>self.particleId = "<STR_LIT>" % (str(self._hsObj._workerID),<EOL>str(Particle._nextParticleID))<EOL>Particle._nextParticleID += <NUM_LIT:1><EOL><DEDENT>self.genIdx = newFromClone['<STR_LIT>']<EOL>self.swarmId = newFromClone['<STR_LIT>']<EOL>_setupVars(flattenedPermuteVars)<EOL>self.initStateFrom(self.particleId, newFromClone, newBest=False)<EOL><DEDENT>else:<EOL><INDENT>assert False, "<STR_LIT>"<EOL><DEDENT>self.logger.debug("<STR_LIT>" % (str(self)))<EOL>

Create a particle.

        There are 3 fundamentally different methods of instantiating a particle:
        1.) You can instantiate a new one from scratch, at generation index #0. This
              particle gets a new particleId.
                required: swarmId
                optional: newFarFrom
                must be None: evolveFromState, newFromClone

        2.) You can instantiate one from savedState, in which case it's generation
              index is incremented (from the value stored in the saved state) and
              its particleId remains the same.
                required: evolveFromState
                optional:
                must be None: flattenedPermuteVars, swarmId, newFromClone

        3.) You can clone another particle, creating a new particle at the same
              generationIdx but a different particleId. This new particle will end
              up at exactly the same position as the one it was cloned from. If
              you want to move it to the next position, or just jiggle it a bit, call
              newPosition() or agitate() after instantiation.
                required: newFromClone
                optional:
                must be None: flattenedPermuteVars, swarmId, evolveFromState

        Parameters:
        --------------------------------------------------------------------
        hsObj:    The HypersearchV2 instance

        resultsDB: the ResultsDB instance that holds all the model results

        flattenedPermuteVars: dict() containing the (key, PermuteVariable) pairs
              of the  flattened permutation variables as read from the permutations
              file.

        swarmId: String that represents the encoder names of the encoders that are
              to be included in this particle's model. Of the form
              'encoder1.encoder2'.
              Required for creation method #1.

        newFarFrom: If not None, this is a list of other particleState dicts in the
              swarm that we want to be as far away from as possible. Optional
              argument for creation method #1.

        evolveFromState: If not None, evolve an existing particle. This is a
              dict containing the particle's state. Preserve the particleId, but
              increment the generation index. Required for creation method #2.

        newFromClone: If not None, clone this other particle's position and generation
              index, with small random perturbations. This is a dict containing the
              particle's state. Required for creation method #3.

        newParticleId: Only applicable when newFromClone is True. Give the clone
              a new particle ID.


varStates = dict()<EOL>for varName, var in self.permuteVars.iteritems():<EOL><INDENT>varStates[varName] = var.getState()<EOL><DEDENT>return dict(id=self.particleId,<EOL>genIdx=self.genIdx,<EOL>swarmId=self.swarmId,<EOL>varStates=varStates)<EOL>

Get the particle state as a dict. This is enough information to
        instantiate this particle on another worker.


def initStateFrom(self, particleId, particleState, newBest):

<EOL>if newBest:<EOL><INDENT>(bestResult, bestPosition) = self._resultsDB.getParticleBest(particleId)<EOL><DEDENT>else:<EOL><INDENT>bestResult = bestPosition = None<EOL><DEDENT>varStates = particleState['<STR_LIT>']<EOL>for varName in varStates.keys():<EOL><INDENT>varState = copy.deepcopy(varStates[varName])<EOL>if newBest:<EOL><INDENT>varState['<STR_LIT>'] = bestResult<EOL><DEDENT>if bestPosition is not None:<EOL><INDENT>varState['<STR_LIT>'] = bestPosition[varName]<EOL><DEDENT>self.permuteVars[varName].setState(varState)<EOL><DEDENT>

Init all of our variable positions, velocities, and optionally the best
        result and best position from the given particle.

        If newBest is true, we get the best result and position for this new
        generation from the resultsDB, This is used when evoloving a particle
        because the bestResult and position as stored in was the best AT THE TIME
        THAT PARTICLE STARTED TO RUN and does not include the best since that
        particle completed.


<EOL>allowedToMove = True<EOL>for varName in particleState['<STR_LIT>']:<EOL><INDENT>if '<STR_LIT::>' in varName:  <EOL><INDENT>if varName not in self.permuteVars:<EOL><INDENT>continue<EOL><DEDENT>state = copy.deepcopy(particleState['<STR_LIT>'][varName])<EOL>state['<STR_LIT>'] = state['<STR_LIT>']<EOL>state['<STR_LIT>'] = state['<STR_LIT>']<EOL>if not allowedToMove:<EOL><INDENT>state['<STR_LIT>'] = <NUM_LIT:0><EOL><DEDENT>self.permuteVars[varName].setState(state)<EOL>if allowedToMove:<EOL><INDENT>self.permuteVars[varName].resetVelocity(self._rng)<EOL><DEDENT><DEDENT><DEDENT>

Copy all encoder variables from particleState into this particle.

        Parameters:
        --------------------------------------------------------------
        particleState:        dict produced by a particle's getState() method


def copyVarStatesFrom(self, particleState, varNames):

<EOL>allowedToMove = True<EOL>for varName in particleState['<STR_LIT>']:<EOL><INDENT>if varName in varNames:<EOL><INDENT>if varName not in self.permuteVars:<EOL><INDENT>continue<EOL><DEDENT>state = copy.deepcopy(particleState['<STR_LIT>'][varName])<EOL>state['<STR_LIT>'] = state['<STR_LIT>']<EOL>state['<STR_LIT>'] = state['<STR_LIT>']<EOL>if not allowedToMove:<EOL><INDENT>state['<STR_LIT>'] = <NUM_LIT:0><EOL><DEDENT>self.permuteVars[varName].setState(state)<EOL>if allowedToMove:<EOL><INDENT>self.permuteVars[varName].resetVelocity(self._rng)<EOL><DEDENT><DEDENT><DEDENT>

Copy specific variables from particleState into this particle.

        Parameters:
        --------------------------------------------------------------
        particleState:        dict produced by a particle's getState() method
        varNames:             which variables to copy


result = dict()<EOL>for (varName, value) in self.permuteVars.iteritems():<EOL><INDENT>result[varName] = value.getPosition()<EOL><DEDENT>return result<EOL>

Return the position of this particle. This returns a dict() of key
        value pairs where each key is the name of the flattened permutation
        variable and the value is its chosen value.

        Parameters:
        --------------------------------------------------------------
        retval:     dict() of flattened permutation choices


result = dict()<EOL>for (varName, value) in pState['<STR_LIT>'].iteritems():<EOL><INDENT>result[varName] = value['<STR_LIT>']<EOL><DEDENT>return result<EOL>

Return the position of a particle given its state dict.

        Parameters:
        --------------------------------------------------------------
        retval:     dict() of particle position, keys are the variable names,
                      values are their positions


for (varName, var) in self.permuteVars.iteritems():<EOL><INDENT>var.agitate()<EOL><DEDENT>self.newPosition()<EOL>

Agitate this particle so that it is likely to go to a new position.
        Every time agitate is called, the particle is jiggled an even greater
        amount.

        Parameters:
        --------------------------------------------------------------
        retval:               None


<EOL>globalBestPosition = None<EOL>if self._hsObj._speculativeParticles:<EOL><INDENT>genIdx = self.genIdx<EOL><DEDENT>else:<EOL><INDENT>genIdx = self.genIdx - <NUM_LIT:1><EOL><DEDENT>if genIdx >= <NUM_LIT:0>:<EOL><INDENT>(bestModelId, _) = self._resultsDB.bestModelIdAndErrScore(self.swarmId,<EOL>genIdx)<EOL>if bestModelId is not None:<EOL><INDENT>(particleState, _, _, _, _) = self._resultsDB.getParticleInfo(<EOL>bestModelId)<EOL>globalBestPosition = Particle.getPositionFromState(particleState)<EOL><DEDENT><DEDENT>for (varName, var) in self.permuteVars.iteritems():<EOL><INDENT>if whichVars is not None and varName not in whichVars:<EOL><INDENT>continue<EOL><DEDENT>if globalBestPosition is None:<EOL><INDENT>var.newPosition(None, self._rng)<EOL><DEDENT>else:<EOL><INDENT>var.newPosition(globalBestPosition[varName], self._rng)<EOL><DEDENT><DEDENT>position = self.getPosition()<EOL>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>msg = StringIO.StringIO()<EOL>print >> msg, "<STR_LIT>" % (pprint.pformat(position,<EOL>indent=<NUM_LIT:4>))<EOL>print >> msg, "<STR_LIT>"<EOL>for (varName, var) in self.permuteVars.iteritems():<EOL><INDENT>print >> msg, "<STR_LIT>" % (varName, str(var))<EOL><DEDENT>self.logger.debug(msg.getvalue())<EOL>msg.close()<EOL><DEDENT>return position<EOL>

Choose a new position based on results obtained so far from all other
        particles.

        Parameters:
        --------------------------------------------------------------
        whichVars:       If not None, only move these variables
        retval:               new position


<EOL>self._hsObj = hsObj<EOL>self.logger = self._hsObj.logger<EOL>self._state = None<EOL>self._priorStateJSON = None<EOL>self._dirty = False<EOL>self.readStateFromDB()<EOL>

Create our state object.

        Parameters:
        ---------------------------------------------------------------------
        hsObj:     Reference to the HypersesarchV2 instance
        cjDAO:     ClientJobsDAO instance
        logger:    logger to use
        jobID:     our JobID


return self._dirty<EOL>

Return true if our local copy of the state has changed since the
        last time we read from the DB.


return self._state['<STR_LIT>']<EOL>

Return true if the search should be considered over.