tm = object.__new__(cls)<EOL>tm.columnDimensions = tuple(proto.columnDimensions)<EOL>tm.cellsPerColumn = int(proto.cellsPerColumn)<EOL>tm.activationThreshold = int(proto.activationThreshold)<EOL>tm.initialPermanence = round(proto.initialPermanence, EPSILON_ROUND)<EOL>tm.connectedPermanence = round(proto.connectedPermanence, EPSILON_ROUND)<EOL>tm.minThreshold = int(proto.minThreshold)<EOL>tm.maxNewSynapseCount = int(proto.maxNewSynapseCount)<EOL>tm.permanenceIncrement = round(proto.permanenceIncrement, EPSILON_ROUND)<EOL>tm.permanenceDecrement = round(proto.permanenceDecrement, EPSILON_ROUND)<EOL>tm.predictedSegmentDecrement = round(proto.predictedSegmentDecrement,<EOL>EPSILON_ROUND)<EOL>tm.maxSegmentsPerCell = int(proto.maxSegmentsPerCell)<EOL>tm.maxSynapsesPerSegment = int(proto.maxSynapsesPerSegment)<EOL>tm.connections = Connections.read(proto.connections)<EOL>tm._random = Random()<EOL>tm._random.read(proto.random)<EOL>tm.activeCells = [int(x) for x in proto.activeCells]<EOL>tm.winnerCells = [int(x) for x in proto.winnerCells]<EOL>flatListLength = tm.connections.segmentFlatListLength()<EOL>tm.numActiveConnectedSynapsesForSegment = [<NUM_LIT:0>] * flatListLength<EOL>tm.numActivePotentialSynapsesForSegment = [<NUM_LIT:0>] * flatListLength<EOL>tm.lastUsedIterationForSegment = [<NUM_LIT:0>] * flatListLength<EOL>tm.activeSegments = []<EOL>tm.matchingSegments = []<EOL>for protoSegment in proto.activeSegments:<EOL><INDENT>tm.activeSegments.append(<EOL>tm.connections.getSegment(protoSegment.cell,<EOL>protoSegment.idxOnCell))<EOL><DEDENT>for protoSegment in proto.matchingSegments:<EOL><INDENT>tm.matchingSegments.append(<EOL>tm.connections.getSegment(protoSegment.cell,<EOL>protoSegment.idxOnCell))<EOL><DEDENT>for protoSegment in proto.numActivePotentialSynapsesForSegment:<EOL><INDENT>segment = tm.connections.getSegment(protoSegment.cell,<EOL>protoSegment.idxOnCell)<EOL>tm.numActivePotentialSynapsesForSegment[segment.flatIdx] = (<EOL>int(protoSegment.number))<EOL><DEDENT>tm.iteration = long(proto.iteration)<EOL>for protoSegment in proto.lastUsedIterationForSegment:<EOL><INDENT>segment = tm.connections.getSegment(protoSegment.cell,<EOL>protoSegment.idxOnCell)<EOL>tm.lastUsedIterationForSegment[segment.flatIdx] = (<EOL>long(protoSegment.number))<EOL><DEDENT>return tm<EOL>

Reads deserialized data from proto object.

:param proto: (DynamicStructBuilder) Proto object

:returns: (:class:TemporalMemory) TemporalMemory instance


def compute(self, activeColumns, learn=True):

self.activateCells(sorted(activeColumns), learn)<EOL>self.activateDendrites(learn)<EOL>

Perform one time step of the Temporal Memory algorithm.

This method calls :meth:`activateCells`, then calls
:meth:`activateDendrites`. Using :class:`TemporalMemory` via its
:meth:`compute` method ensures that you'll always be able to call
:meth:`getPredictiveCells` to get predictions for the next time step.

:param activeColumns: (iter) Indices of active columns.

:param learn: (bool) Whether or not learning is enabled.


self.activeCells = []<EOL>self.winnerCells = []<EOL>self.activeSegments = []<EOL>self.matchingSegments = []<EOL>

Indicates the start of a new sequence. Clears any predictions and makes sure
synapses don't grow to the currently active cells in the next time step.


if cell >= self.numberOfCells() or cell < <NUM_LIT:0>:<EOL><INDENT>raise IndexError("<STR_LIT>")<EOL><DEDENT>

Raises an error if cell index is invalid.

:param cell: (int) Cell index


return self.cellsPerColumn<EOL>

Returns the number of cells per column.

:returns: (int) The number of cells per column.


@classmethod<EOL><INDENT>def _leastUsedCell(cls, random, cells, connections):<DEDENT>

leastUsedCells = []<EOL>minNumSegments = float("<STR_LIT>")<EOL>for cell in cells:<EOL><INDENT>numSegments = connections.numSegments(cell)<EOL>if numSegments < minNumSegments:<EOL><INDENT>minNumSegments = numSegments<EOL>leastUsedCells = []<EOL><DEDENT>if numSegments == minNumSegments:<EOL><INDENT>leastUsedCells.append(cell)<EOL><DEDENT><DEDENT>i = random.getUInt32(len(leastUsedCells))<EOL>return leastUsedCells[i]<EOL>

Gets the cell with the smallest number of segments.
Break ties randomly.

:param random: (Object)
Random number generator. Gets mutated.

:param cells: (list)
Indices of cells.

:param connections: (Object)
Connections instance for the TM.

:returns: (int) Cell index.


return self.numberOfColumns() * self.cellsPerColumn<EOL>

Returns the number of cells in this layer.

:returns: (int) Number of cells


def punishPredictedColumn(self, column, columnActiveSegments,<EOL>columnMatchingSegments, prevActiveCells,<EOL>prevWinnerCells):

self._punishPredictedColumn(<EOL>self.connections, columnMatchingSegments, prevActiveCells,<EOL>self.predictedSegmentDecrement)<EOL>

Punishes the Segments that incorrectly predicted a column to be active.

:param column: (int) Index of bursting column.

:param columnActiveSegments: (iter) Active segments for this column, or None
       if there aren't any.

:param columnMatchingSegments: (iter) Matching segments for this column, or
       None if there aren't any.

:param prevActiveCells: (list) Active cells in ``t-1``.

:param prevWinnerCells: (list) Winner cells in ``t-1``.


return [cls.getCellIndex(c) for c in cells]<EOL>

Returns the indices of the cells passed in.

:param cells: (list) cells to find the indices of


if column >= self.numberOfColumns() or column < <NUM_LIT:0>:<EOL><INDENT>raise IndexError("<STR_LIT>")<EOL><DEDENT>

Raises an error if column index is invalid.

:param column: (int) Column index


self.initialPermanence = initialPermanence<EOL>

Sets the initial permanence.

:param initialPermanence: (float) The initial permanence.


self._mmTraces = {}<EOL>self._mmData = {}<EOL>


assert len(traces) > <NUM_LIT:0>, "<STR_LIT>"<EOL>table = PrettyTable(["<STR_LIT:#>"] + [trace.prettyPrintTitle() for trace in traces])<EOL>for i in xrange(len(traces[<NUM_LIT:0>].data)):<EOL><INDENT>if breakOnResets and breakOnResets.data[i]:<EOL><INDENT>table.add_row(["<STR_LIT>"] * (len(traces) + <NUM_LIT:1>))<EOL><DEDENT>table.add_row([i] +<EOL>[trace.prettyPrintDatum(trace.data[i]) for trace in traces])<EOL><DEDENT>return table.get_string().encode("<STR_LIT:utf-8>")<EOL>

Returns pretty-printed table of traces.

@param traces (list) Traces to print in table
@param breakOnResets (BoolsTrace) Trace of resets to break table on

@return (string) Pretty-printed table of traces.


def __init__(self, monitor, title, show=True):

self._monitor = monitor<EOL>self._title = title<EOL>self._fig = self._initFigure()<EOL>self._show = show<EOL>if self._show:<EOL><INDENT>plt.ion()<EOL>plt.show()<EOL><DEDENT>

@param monitor (MonitorMixinBase) Monitor Mixin instance that generated
                                  this plot

@param title  (string)            Plot title


def addGraph(self, data, position=<NUM_LIT>, xlabel=None, ylabel=None):

ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)<EOL>ax.plot(data)<EOL>plt.draw()<EOL>

Adds a graph to the plot's figure.

        @param data See matplotlib.Axes.plot documentation.
        @param position A 3-digit number. The first two digits define a 2D grid
                where subplots may be added. The final digit specifies the nth grid
                location for the added subplot
        @param xlabel text to be displayed on the x-axis
        @param ylabel text to be displayed on the y-axis


def add2DArray(self, data, position=<NUM_LIT>, xlabel=None, ylabel=None, cmap=None,<EOL>aspect="<STR_LIT>", interpolation="<STR_LIT>", name=None):

if cmap is None:<EOL><INDENT>cmap = cm.Greys<EOL><DEDENT>ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)<EOL>ax.imshow(data, cmap=cmap, aspect=aspect, interpolation=interpolation)<EOL>if self._show:<EOL><INDENT>plt.draw()<EOL><DEDENT>if name is not None:<EOL><INDENT>if not os.path.exists("<STR_LIT>"):<EOL><INDENT>os.mkdir("<STR_LIT>")<EOL><DEDENT>plt.savefig("<STR_LIT>".format(name=name), bbox_inches="<STR_LIT>",<EOL>figsize=(<NUM_LIT:8>, <NUM_LIT:6>), dpi=<NUM_LIT>)<EOL><DEDENT>

Adds an image to the plot's figure.

        @param data a 2D array. See matplotlib.Axes.imshow documentation.
        @param position A 3-digit number. The first two digits define a 2D grid
                where subplots may be added. The final digit specifies the nth grid
                location for the added subplot
        @param xlabel text to be displayed on the x-axis
        @param ylabel text to be displayed on the y-axis
        @param cmap color map used in the rendering
        @param aspect how aspect ratio is handled during resize
        @param interpolation interpolation method


return self._mmTraces["<STR_LIT>"]<EOL>


self._mmComputeTransitionTraces()<EOL>return self._mmTraces["<STR_LIT>"]<EOL>

@return (Trace) Trace of predicted => active columns


@return (Trace) Trace of predicted => inactive columns


self._mmComputeTransitionTraces()<EOL>numSequencesForCell = defaultdict(lambda: <NUM_LIT:0>)<EOL>for predictedActiveCells in (<EOL>self._mmData["<STR_LIT>"].values()):<EOL><INDENT>for cell in predictedActiveCells:<EOL><INDENT>numSequencesForCell[cell] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return Metric(self,<EOL>"<STR_LIT>",<EOL>numSequencesForCell.values())<EOL>

Metric for number of sequences each predicted => active cell appears in

Note: This metric is flawed when it comes to high-order sequences.

@return (Metric) metric


def mmGetCellActivityPlot(self, title="<STR_LIT>", showReset=False,<EOL>resetShading=<NUM_LIT>, activityType="<STR_LIT>"):

if activityType == "<STR_LIT>":<EOL><INDENT>self._mmComputeTransitionTraces()<EOL><DEDENT>cellTrace = copy.deepcopy(self._mmTraces[activityType].data)<EOL>for i in xrange(len(cellTrace)):<EOL><INDENT>cellTrace[i] = self.getCellIndices(cellTrace[i])<EOL><DEDENT>return self.mmGetCellTracePlot(cellTrace, self.numberOfCells(),<EOL>activityType, title, showReset,<EOL>resetShading)<EOL>

Returns plot of the cell activity.

@param title        (string)  an optional title for the figure

@param showReset    (bool)    if true, the first set of cell activities
                              after a reset will have a gray background

@param resetShading (float)   if showReset is true, this float specifies the
                              intensity of the reset background with 0.0
                              being white and 1.0 being black

@param activityType (string)  The type of cell activity to display. Valid
                              types include "activeCells",
                              "predictiveCells", "predictedCells",
                              and "predictedActiveCells"

@return (Plot) plot


return self._mmTraces["<STR_LIT>"]<EOL>

@return (Trace) Trace of sequence labels


return self._mmTraces["<STR_LIT>"]<EOL>

@return (Trace) Trace of predictive cells


return Metric.createFromTrace(trace.makeCountsTrace(),<EOL>excludeResets=self.mmGetTraceResets())<EOL>

Convenience method to compute a metric over an indices trace, excluding
resets.

@param (IndicesTrace) Trace of indices

@return (Metric) Metric over trace excluding resets


self._mmComputeTransitionTraces()<EOL>numCellsPerColumn = []<EOL>for predictedActiveCells in (<EOL>self._mmData["<STR_LIT>"].values()):<EOL><INDENT>cellsForColumn = self.mapCellsToColumns(predictedActiveCells)<EOL>numCellsPerColumn += [len(x) for x in cellsForColumn.values()]<EOL><DEDENT>return Metric(self,<EOL>"<STR_LIT>",<EOL>numCellsPerColumn)<EOL>

Metric for number of predicted => active cells per column for each sequence

@return (Metric) metric


@return (Trace) Trace of predicted => inactive cells


text = "<STR_LIT>"<EOL>text += ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>text += "<STR_LIT>"<EOL>columns = range(self.numberOfColumns())<EOL>for column in columns:<EOL><INDENT>cells = self.cellsForColumn(column)<EOL>for cell in cells:<EOL><INDENT>segmentDict = dict()<EOL>for seg in self.connections.segmentsForCell(cell):<EOL><INDENT>synapseList = []<EOL>for synapse in self.connections.synapsesForSegment(seg):<EOL><INDENT>synapseData = self.connections.dataForSynapse(synapse)<EOL>synapseList.append(<EOL>(synapseData.presynapticCell, synapseData.permanence))<EOL><DEDENT>synapseList.sort()<EOL>synapseStringList = ["<STR_LIT>".format(sourceCell, permanence) for<EOL>sourceCell, permanence in synapseList]<EOL>segmentDict[seg] = "<STR_LIT>".format("<STR_LIT:U+0020>".join(synapseStringList))<EOL><DEDENT>text += ("<STR_LIT>".format(<EOL>column, cell,<EOL>len(segmentDict.values()),<EOL>"<STR_LIT>".format("<STR_LIT:U+002CU+0020>".join(segmentDict.values()))))<EOL><DEDENT>if column < len(columns) - <NUM_LIT:1>:  <EOL><INDENT>text += "<STR_LIT:\n>"<EOL><DEDENT><DEDENT>text += "<STR_LIT>"<EOL>return text<EOL>

Pretty print the connections in the temporal memory.

TODO: Use PrettyTable.

@return (string) Pretty-printed text


trace = CountsTrace(self.monitor, "<STR_LIT>".format(self.title))<EOL>countsTrace = self.makeCountsTrace()<EOL>def accumulate(iterator):<EOL><INDENT>total = <NUM_LIT:0><EOL>for item in iterator:<EOL><INDENT>total += item<EOL>yield total<EOL><DEDENT><DEDENT>trace.data = list(accumulate(countsTrace.data))<EOL>return trace<EOL>

@return (CountsTrace) A new Trace made up of cumulative counts of this
trace's indices.


return str(datum) if datum is not None else "<STR_LIT>"<EOL>

@param datum (object) Datum from `self.data` to pretty-print

@return (string) Pretty-printed datum


trace = CountsTrace(self.monitor, "<STR_LIT>".format(self.title))<EOL>trace.data = [len(indices) for indices in self.data]<EOL>return trace<EOL>

@return (CountsTrace) A new Trace made up of counts of this trace's indices.


global g_models<EOL>data = json.loads(web.data())<EOL>modelParams = data["<STR_LIT>"]<EOL>predictedFieldName = data["<STR_LIT>"]<EOL>if name in g_models.keys():<EOL><INDENT>raise web.badrequest("<STR_LIT>" % name)<EOL><DEDENT>model = ModelFactory.create(modelParams)<EOL>model.enableInference({'<STR_LIT>': predictedFieldName})<EOL>g_models[name] = model<EOL>return json.dumps({"<STR_LIT:success>": name})<EOL>

/models/{name}

schema:
{
  "modelParams": dict containing model parameters
  "predictedFieldName": str
}

returns:
{"success":name}


return self._particleBest.get(particleId, (None, None))<EOL>

Return the best score and position for a given particle. The position
        is given as a dict, with varName:varPosition items in it.

        Parameters:
        ---------------------------------------------------------------------
        particleId:    which particle
        retval:        (bestResult, bestPosition)


def __init__(self, searchParams, workerID=None, cjDAO=None, jobID=None,<EOL>logLevel=None):

<EOL>self.logger = logging.getLogger("<STR_LIT:.>".join( ['<STR_LIT>',<EOL>self.__class__.__module__, self.__class__.__name__]))<EOL>if logLevel is not None:<EOL><INDENT>self.logger.setLevel(logLevel)<EOL><DEDENT>random.seed(<NUM_LIT>)<EOL>self._searchParams = searchParams<EOL>self._workerID = workerID<EOL>self._cjDAO = cjDAO<EOL>self._jobID = jobID<EOL>self.logger.info("<STR_LIT>" % (pprint.pformat(<EOL>clippedObj(searchParams))))<EOL>self._createCheckpoints = self._searchParams.get('<STR_LIT>',<EOL>False)<EOL>self._maxModels = self._searchParams.get('<STR_LIT>', None)<EOL>if self._maxModels == -<NUM_LIT:1>:<EOL><INDENT>self._maxModels = None<EOL><DEDENT>self._predictionCacheMaxRecords = self._searchParams.get('<STR_LIT>', None)<EOL>self._speculativeParticles = self._searchParams.get('<STR_LIT>',<EOL>bool(int(Configuration.get(<EOL>'<STR_LIT>'))))<EOL>self._speculativeWaitSecondsMax = float(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._maxBranching= int(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._minFieldContribution= float(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._jobCancelled = False<EOL>if '<STR_LIT>' in self._searchParams:<EOL><INDENT>useTerminators = self._searchParams['<STR_LIT>']<EOL>useTerminators = str(int(useTerminators))<EOL>Configuration.set('<STR_LIT>', useTerminators)<EOL>Configuration.set('<STR_LIT>', useTerminators)<EOL>Configuration.set('<STR_LIT>', useTerminators)<EOL><DEDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>self._maxModels = int(os.environ['<STR_LIT>'])<EOL><DEDENT>self._dummyModel = self._searchParams.get('<STR_LIT>', None)<EOL>self._tempDir = None<EOL>try:<EOL><INDENT>if '<STR_LIT:description>' in self._searchParams:<EOL><INDENT>if ('<STR_LIT>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams):<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>searchParamObj = self._searchParams<EOL>anomalyParams = searchParamObj['<STR_LIT:description>'].get('<STR_LIT>',<EOL>dict())<EOL>if anomalyParams is None:<EOL><INDENT>anomalyParams = dict()<EOL><DEDENT>if (('<STR_LIT>' not in anomalyParams) or<EOL>(anomalyParams['<STR_LIT>'] is None)):<EOL><INDENT>streamDef = self._getStreamDef(searchParamObj['<STR_LIT:description>'])<EOL>from nupic.data.stream_reader import StreamReader<EOL>try:<EOL><INDENT>streamReader = StreamReader(streamDef, isBlocking=False,<EOL>maxTimeout=<NUM_LIT:0>, eofOnTimeout=True)<EOL>anomalyParams['<STR_LIT>'] =streamReader.getDataRowCount()<EOL><DEDENT>except Exception:<EOL><INDENT>anomalyParams['<STR_LIT>'] = None<EOL><DEDENT>self._searchParams['<STR_LIT:description>']['<STR_LIT>'] = anomalyParams<EOL><DEDENT>outDir = self._tempDir = tempfile.mkdtemp()<EOL>expGenerator([<EOL>'<STR_LIT>' % (<EOL>json.dumps(self._searchParams['<STR_LIT:description>'])),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % (outDir)])<EOL>permutationsScript = os.path.join(outDir, '<STR_LIT>')<EOL><DEDENT>elif '<STR_LIT>' in self._searchParams:<EOL><INDENT>if ('<STR_LIT:description>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams):<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>permutationsScript = self._searchParams['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in self._searchParams:<EOL><INDENT>if ('<STR_LIT:description>' in self._searchParams or<EOL>'<STR_LIT>' in self._searchParams):<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>assert ('<STR_LIT>' in self._searchParams)<EOL>outDir = self._tempDir = tempfile.mkdtemp()<EOL>permutationsScript = os.path.join(outDir, '<STR_LIT>')<EOL>fd = open(permutationsScript, '<STR_LIT:w>')<EOL>fd.write(self._searchParams['<STR_LIT>'])<EOL>fd.close()<EOL>fd = open(os.path.join(outDir, '<STR_LIT>'), '<STR_LIT:w>')<EOL>fd.write(self._searchParams['<STR_LIT>'])<EOL>fd.close()<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._basePath = os.path.dirname(permutationsScript)<EOL>self._baseDescription = open(os.path.join(self._basePath,<EOL>'<STR_LIT>')).read()<EOL>self._baseDescriptionHash = hashlib.md5(self._baseDescription).digest()<EOL>modelDescription, _ = helpers.loadExperiment(self._basePath)<EOL>self._readPermutationsFile(permutationsScript, modelDescription)<EOL>if self._cjDAO is not None:<EOL><INDENT>updated = self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=None,<EOL>newValue = self._baseDescription)<EOL>if updated:<EOL><INDENT>permContents = open(permutationsScript).read()<EOL>self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=None,<EOL>newValue = permContents)<EOL><DEDENT><DEDENT>if self._dummyModelParamsFunc is not None:<EOL><INDENT>if self._dummyModel is None:<EOL><INDENT>self._dummyModel = dict()<EOL><DEDENT><DEDENT>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>msg = io.StringIO()<EOL>print("<STR_LIT>", file=msg)<EOL>info = dict()<EOL>for key in ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>info[key] = getattr(self, key)<EOL><DEDENT>print(pprint.pformat(info), file=msg)<EOL>self.logger.debug(msg.getvalue())<EOL>msg.close()<EOL><DEDENT>self._resultsDB = ResultsDB(self)<EOL>self._swarmTerminator = SwarmTerminator()<EOL>self._hsState = None<EOL>self._maxUniqueModelAttempts = int(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._modelOrphanIntervalSecs = float(Configuration.get(<EOL>'<STR_LIT>'))<EOL>self._maxPctErrModels = float(Configuration.get(<EOL>'<STR_LIT>'))<EOL><DEDENT>except:<EOL><INDENT>if self._tempDir is not None:<EOL><INDENT>shutil.rmtree(self._tempDir)<EOL>self._tempDir = None<EOL><DEDENT>raise<EOL><DEDENT>return<EOL>

Instantiate the HyperseachV2 instance.

        Parameters:
        ----------------------------------------------------------------------
        searchParams:   a dict of the job's search parameters. The format is:

          persistentJobGUID:  REQUIRED.
                              Persistent, globally-unique identifier for this job
                              for use in constructing persistent model checkpoint
                              keys. MUST be compatible with S3 key-naming rules, but
                              MUST NOT contain forward slashes. This GUID is
                              expected to retain its global uniqueness across
                              clusters and cluster software updates (unlike the
                              record IDs in the Engine's jobs table, which recycle
                              upon table schema change and software update). In the
                              future, this may also be instrumental for checkpoint
                              garbage collection.

          permutationsPyFilename:
                              OPTIONAL - path to permutations.py file
          permutationsPyContents:
                              OPTIONAL - JSON encoded string with
                                          contents of permutations.py file
          descriptionPyContents:
                              OPTIONAL - JSON encoded string with
                                          contents of base description.py file
          description:        OPTIONAL - JSON description of the search
          createCheckpoints:  OPTIONAL - Whether to create checkpoints
          useTerminators      OPTIONAL - True of False (default config.xml). When set
                                         to False, the model and swarm terminators
                                         are disabled
          maxModels:          OPTIONAL - max # of models to generate
                                        NOTE: This is a deprecated location for this
                                        setting. Now, it should be specified through
                                        the maxModels variable within the permutations
                                        file, or maxModels in the JSON description
          dummyModel:         OPTIONAL - Either (True/False) or a dict of parameters
                                         for a dummy model. If this key is absent,
                                         a real model is trained.
                                         See utils.py/OPFDummyModel runner for the
                                         schema of the dummy parameters
          speculativeParticles OPTIONAL - True or False (default obtained from
                                         nupic.hypersearch.speculative.particles.default
                                         configuration property). See note below.

          NOTE: The caller must provide just ONE of the following to describe the
          hypersearch:
                1.) permutationsPyFilename
            OR  2.) permutationsPyContents & permutationsPyContents
            OR  3.) description

          The schema for the description element can be found at:
           "py/nupic/frameworks/opf/expGenerator/experimentDescriptionSchema.json"

          NOTE about speculativeParticles: If true (not 0), hypersearch workers will
          go ahead and create and run particles in subsequent sprints and
          generations before the current generation or sprint has been completed. If
          false, a worker will wait in a sleep loop until the current generation or
          sprint has finished before choosing the next particle position or going
          into the next sprint. When true, the best model can be found faster, but
          results are less repeatable due to the randomness of when each worker
          completes each particle. This property can be overridden via the
          speculativeParticles element of the Hypersearch job params.

        workerID:   our unique Hypersearch worker ID

        cjDAO:      ClientJobsDB Data Access Object
        jobID:      job ID for this hypersearch job
        logLevel:   override logging level to this value, if not None


if self._tempDir is not None and os.path.isdir(self._tempDir):<EOL><INDENT>self.logger.debug("<STR_LIT>", self._tempDir)<EOL>shutil.rmtree(self._tempDir)<EOL>self._tempDir = None<EOL><DEDENT>return<EOL>


raise NotImplementedError<EOL>

Generates a lookup dictionary of permutation variables whose values
        are too complex for labels, so that artificial labels have to be generated
        for them.

        Parameters:
        ---------------------------------------------------------
        retval:       A look-up dictionary of permutation
                      variables whose values are too complex for labels, so
                      artificial labels were generated instead (e.g., "Choice0",
                      "Choice1", etc.); the key is the name of the complex variable
                      and the value is:
                        dict(labels=<list_of_labels>, values=<list_of_values>).


if self._hsState is None:<EOL><INDENT>self._hsState =  HsState(self)<EOL><DEDENT>self._hsState.readStateFromDB()<EOL>completedSwarms = set()<EOL>if exhaustedSwarmId is not None:<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (exhaustedSwarmId))<EOL>(particles, _, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId=exhaustedSwarmId, matured=False)<EOL>if len(particles) > <NUM_LIT:0>:<EOL><INDENT>exhaustedSwarmStatus = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>exhaustedSwarmStatus = '<STR_LIT>'<EOL><DEDENT><DEDENT>if self._killUselessSwarms:<EOL><INDENT>self._hsState.killUselessSwarms()<EOL><DEDENT>completingSwarms = self._hsState.getCompletingSwarms()<EOL>for swarmId in completingSwarms:<EOL><INDENT>(particles, _, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId=swarmId, matured=False)<EOL>if len(particles) == <NUM_LIT:0>:<EOL><INDENT>completedSwarms.add(swarmId)<EOL><DEDENT><DEDENT>completedSwarmGens = self._resultsDB.getMaturedSwarmGenerations()<EOL>priorCompletedSwarms = self._hsState.getCompletedSwarms()<EOL>for (swarmId, genIdx, errScore) in completedSwarmGens:<EOL><INDENT>if swarmId in priorCompletedSwarms:<EOL><INDENT>continue<EOL><DEDENT>completedList = self._swarmTerminator.recordDataPoint(<EOL>swarmId=swarmId, generation=genIdx, errScore=errScore)<EOL>statusMsg = "<STR_LIT>""<STR_LIT>" % (genIdx, swarmId, errScore)<EOL>if len(completedList) > <NUM_LIT:0>:<EOL><INDENT>statusMsg = "<STR_LIT>" % (statusMsg, completedList)<EOL><DEDENT>self.logger.info(statusMsg)<EOL>self._cjDAO.jobSetFields (jobID=self._jobID,<EOL>fields=dict(engStatus=statusMsg),<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL>if '<STR_LIT>' in os.environ:<EOL><INDENT>while True:<EOL><INDENT>resultsStr = self._cjDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if resultsStr is None:<EOL><INDENT>results = {}<EOL><DEDENT>else:<EOL><INDENT>results = json.loads(resultsStr)<EOL><DEDENT>if not '<STR_LIT>' in results:<EOL><INDENT>results['<STR_LIT>'] = {}<EOL><DEDENT>for swarm in completedList:<EOL><INDENT>if swarm not in results['<STR_LIT>']:<EOL><INDENT>results['<STR_LIT>'][swarm] = (genIdx,<EOL>self._swarmTerminator.swarmScores[swarm])<EOL><DEDENT><DEDENT>newResultsStr = json.dumps(results)<EOL>if newResultsStr == resultsStr:<EOL><INDENT>break<EOL><DEDENT>updated = self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=resultsStr,<EOL>newValue = json.dumps(results))<EOL>if updated:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>if len(completedList) > <NUM_LIT:0>:<EOL><INDENT>for name in completedList:<EOL><INDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT:%s>" % (name, genIdx, errScore))<EOL><DEDENT>completedSwarms = completedSwarms.union(completedList)<EOL><DEDENT><DEDENT>if len(completedSwarms)==<NUM_LIT:0> and (exhaustedSwarmId is None):<EOL><INDENT>return<EOL><DEDENT>while True:<EOL><INDENT>if exhaustedSwarmId is not None:<EOL><INDENT>self._hsState.setSwarmState(exhaustedSwarmId, exhaustedSwarmStatus)<EOL><DEDENT>for swarmId in completedSwarms:<EOL><INDENT>self._hsState.setSwarmState(swarmId, '<STR_LIT>')<EOL><DEDENT>if not self._hsState.isDirty():<EOL><INDENT>return<EOL><DEDENT>success = self._hsState.writeStateToDB()<EOL>if success:<EOL><INDENT>jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobResultsStr is not None:<EOL><INDENT>jobResults = json.loads(jobResultsStr)<EOL>bestModelId = jobResults.get('<STR_LIT>', None)<EOL><DEDENT>else:<EOL><INDENT>bestModelId = None<EOL><DEDENT>for swarmId in list(completedSwarms):<EOL><INDENT>(_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(<EOL>swarmId=swarmId, completed=False)<EOL>if bestModelId in modelIds:<EOL><INDENT>modelIds.remove(bestModelId)<EOL><DEDENT>if len(modelIds) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (swarmId,<EOL>str(modelIds)))<EOL>for modelId in modelIds:<EOL><INDENT>self._cjDAO.modelSetFields(modelId,<EOL>dict(engStop=ClientJobsDAO.STOP_REASON_KILLED),<EOL>ignoreUnchanged = True)<EOL><DEDENT><DEDENT>return<EOL><DEDENT>self._hsState.readStateFromDB()<EOL>self.logger.debug("<STR_LIT>"<EOL>"<STR_LIT>" % (pprint.pformat(self._hsState._state, indent=<NUM_LIT:4>)))<EOL><DEDENT>

Periodically, check to see if we should remove a certain field combination
from evaluation (because it is doing so poorly) or move on to the next
sprint (add in more fields).

This method is called from _getCandidateParticleAndSwarm(), which is called
right before we try and create a new model to run.

Parameters:
-----------------------------------------------------------------------
removeSwarmId:     If not None, force a change to the current set of active
                  swarms by removing this swarm. This is used in situations
                  where we can't find any new unique models to create in
                  this swarm. In these situations, we update the hypersearch
                  state regardless of the timestamp of the last time another
                  worker updated it.


def runModel(self, modelID, jobID, modelParams, modelParamsHash,<EOL>jobsDAO, modelCheckpointGUID):

<EOL>if not self._createCheckpoints:<EOL><INDENT>modelCheckpointGUID = None<EOL><DEDENT>self._resultsDB.update(modelID=modelID,<EOL>modelParams=modelParams,<EOL>modelParamsHash=modelParamsHash,<EOL>metricResult = None,<EOL>completed = False,<EOL>completionReason = None,<EOL>matured = False,<EOL>numRecords = <NUM_LIT:0>)<EOL>structuredParams = modelParams['<STR_LIT>']<EOL>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>self.logger.debug("<STR_LIT>" %(pprint.pformat(modelParams, indent=<NUM_LIT:4>), modelID))<EOL><DEDENT>cpuTimeStart = time.clock()<EOL>logLevel = self.logger.getEffectiveLevel()<EOL>try:<EOL><INDENT>if self._dummyModel is None or self._dummyModel is False:<EOL><INDENT>(cmpReason, cmpMsg) = runModelGivenBaseAndParams(<EOL>modelID=modelID,<EOL>jobID=jobID,<EOL>baseDescription=self._baseDescription,<EOL>params=structuredParams,<EOL>predictedField=self._predictedField,<EOL>reportKeys=self._reportKeys,<EOL>optimizeKey=self._optimizeKey,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=self._predictionCacheMaxRecords)<EOL><DEDENT>else:<EOL><INDENT>dummyParams = dict(self._dummyModel)<EOL>dummyParams['<STR_LIT>'] = structuredParams<EOL>if self._dummyModelParamsFunc is not None:<EOL><INDENT>permInfo = dict(structuredParams)<EOL>permInfo ['<STR_LIT>'] = modelParams['<STR_LIT>']['<STR_LIT>']<EOL>dummyParams.update(self._dummyModelParamsFunc(permInfo))<EOL><DEDENT>(cmpReason, cmpMsg) = runDummyModel(<EOL>modelID=modelID,<EOL>jobID=jobID,<EOL>params=dummyParams,<EOL>predictedField=self._predictedField,<EOL>reportKeys=self._reportKeys,<EOL>optimizeKey=self._optimizeKey,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=self._predictionCacheMaxRecords)<EOL><DEDENT>jobsDAO.modelSetCompleted(modelID,<EOL>completionReason = cmpReason,<EOL>completionMsg = cmpMsg,<EOL>cpuTime = time.clock() - cpuTimeStart)<EOL><DEDENT>except InvalidConnectionException as e:<EOL><INDENT>self.logger.warn("<STR_LIT:%s>", e)<EOL><DEDENT>

Run the given model.

        This runs the model described by 'modelParams'. Periodically, it updates
        the results seen on the model to the model database using the databaseAO
        (database Access Object) methods.

        Parameters:
        -------------------------------------------------------------------------
        modelID:             ID of this model in models table

        jobID:               ID for this hypersearch job in the jobs table

        modelParams:         parameters of this specific model
                             modelParams is a dictionary containing the name/value
                             pairs of each variable we are permuting over. Note that
                             variables within an encoder spec have their name
                             structure as:
                               <encoderName>.<encodrVarName>

        modelParamsHash:     hash of modelParamValues

        jobsDAO              jobs data access object - the interface to the jobs
                              database where model information is stored

        modelCheckpointGUID: A persistent, globally-unique identifier for
                              constructing the model checkpoint key


return -<NUM_LIT:1><EOL>

Computes the number of models that are expected to complete as part of
        this instances's HyperSearch.

        NOTE: This is compute-intensive for HyperSearches with a huge number of
        combinations.

        NOTE/TODO:  THIS ONLY WORKS FOR RONOMATIC: This method is exposed for the
                    benefit of perutations_runner.py for use in progress
                    reporting.

        Parameters:
        ---------------------------------------------------------
        retval:       The total number of expected models, if known; -1 if unknown


return None<EOL>

Generates a list of model names that are expected to complete as part of
        this instances's HyperSearch.

        NOTE: This is compute-intensive for HyperSearches with a huge number of
        combinations.

        NOTE/TODO:  THIS ONLY WORKS FOR RONOMATIC: This method is exposed for the
                    benefit of perutations_runner.py.

        Parameters:
        ---------------------------------------------------------
        retval:       List of model names for this HypersearchV2 instance, or
                      None of not applicable


def numModels(self, swarmId=None, includeHidden=False):

<EOL>if includeHidden:<EOL><INDENT>if swarmId is None:<EOL><INDENT>return len(self._allResults)<EOL><DEDENT>else:<EOL><INDENT>return len(self._swarmIdToIndexes.get(swarmId, []))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if swarmId is None:<EOL><INDENT>entries = self._allResults<EOL><DEDENT>else:<EOL><INDENT>entries = [self._allResults[entryIdx]<EOL>for entryIdx in self._swarmIdToIndexes.get(swarmId,[])]<EOL><DEDENT>return len([entry for entry in entries if not entry['<STR_LIT>']])<EOL><DEDENT>

Return the total # of models we have in our database (if swarmId is
        None) or in a specific swarm.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:        A string representation of the sorted list of encoders
                        in this swarm. For example '__address_encoder.__gym_encoder'
        includeHidden:  If False, this will only return the number of models
                        that are not hidden (i.e. orphanned, etc.)
        retval:  numModels


return (self._optimizeKey, self._maximize)<EOL>

Retrives the optimization key name and optimization function.

        Parameters:
        ---------------------------------------------------------
        retval:       (optimizationMetricKey, maximize)
                      optimizationMetricKey: which report key to optimize for
                      maximize: True if we should try and maximize the optimizeKey
                        metric. False if we should minimize it.


def firstNonFullGeneration(self, swarmId, minNumParticles):

if not swarmId in self._swarmNumParticlesPerGeneration:<EOL><INDENT>return None<EOL><DEDENT>numPsPerGen = self._swarmNumParticlesPerGeneration[swarmId]<EOL>numPsPerGen = numpy.array(numPsPerGen)<EOL>firstNonFull = numpy.where(numPsPerGen < minNumParticles)[<NUM_LIT:0>]<EOL>if len(firstNonFull) == <NUM_LIT:0>:<EOL><INDENT>return len(numPsPerGen)<EOL><DEDENT>else:<EOL><INDENT>return firstNonFull[<NUM_LIT:0>]<EOL><DEDENT>

Return the generation index of the first generation in the given
        swarm that does not have numParticles particles in it, either still in the
        running state or completed. This does not include orphaned particles.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'
        minNumParticles: minium number of partices required for a full
                      generation.

        retval:  generation index, or None if no particles at all.


<EOL>aggregationPeriod = {<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>}<EOL>aggFunctionsDict = {}<EOL>if '<STR_LIT>' in modelDescription['<STR_LIT>']:<EOL><INDENT>for key in list(aggregationPeriod.keys()):<EOL><INDENT>if key in modelDescription['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>aggregationPeriod[key] = modelDescription['<STR_LIT>']['<STR_LIT>'][key]<EOL><DEDENT><DEDENT>if '<STR_LIT>' in modelDescription['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>for (fieldName, func) in modelDescription['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>aggFunctionsDict[fieldName] = str(func)<EOL><DEDENT><DEDENT><DEDENT>hasAggregation = False<EOL>for v in list(aggregationPeriod.values()):<EOL><INDENT>if v != <NUM_LIT:0>:<EOL><INDENT>hasAggregation = True<EOL>break<EOL><DEDENT><DEDENT>aggFunctionList = list(aggFunctionsDict.items())<EOL>aggregationInfo = dict(aggregationPeriod)<EOL>aggregationInfo['<STR_LIT>'] = aggFunctionList<EOL>streamDef = copy.deepcopy(modelDescription['<STR_LIT>'])<EOL>streamDef['<STR_LIT>'] = copy.deepcopy(aggregationInfo)<EOL>return streamDef<EOL>


self._hsObj = hsObj<EOL>self._allResults = []<EOL>self._errModels = set()<EOL>self._numErrModels = <NUM_LIT:0><EOL>self._completedModels = set()<EOL>self._numCompletedModels = <NUM_LIT:0><EOL>self._modelIDToIdx = dict()<EOL>self._bestResult = numpy.inf<EOL>self._bestModelID = None<EOL>self._swarmBestOverall = dict()<EOL>self._swarmNumParticlesPerGeneration = dict()<EOL>self._modifiedSwarmGens = set()<EOL>self._maturedSwarmGens = set()<EOL>self._particleBest = dict()<EOL>self._particleLatestGenIdx = dict()<EOL>self._swarmIdToIndexes = dict()<EOL>self._paramsHashToIndexes = dict()<EOL>

Instantiate our results database

        Parameters:
        --------------------------------------------------------------------
        hsObj:        Reference to the HypersearchV2 instance


def getParticleInfos(self, swarmId=None, genIdx=None, completed=None,<EOL>matured=None, lastDescendent=False):

<EOL>if swarmId is not None:<EOL><INDENT>entryIdxs = self._swarmIdToIndexes.get(swarmId, [])<EOL><DEDENT>else:<EOL><INDENT>entryIdxs = list(range(len(self._allResults)))<EOL><DEDENT>if len(entryIdxs) == <NUM_LIT:0>:<EOL><INDENT>return ([], [], [], [], [])<EOL><DEDENT>particleStates = []<EOL>modelIds = []<EOL>errScores = []<EOL>completedFlags = []<EOL>maturedFlags = []<EOL>for idx in entryIdxs:<EOL><INDENT>entry = self._allResults[idx]<EOL>if swarmId is not None:<EOL><INDENT>assert (not entry['<STR_LIT>'])<EOL><DEDENT>modelParams = entry['<STR_LIT>']<EOL>isCompleted = entry['<STR_LIT>']<EOL>isMatured = entry['<STR_LIT>']<EOL>particleState = modelParams['<STR_LIT>']<EOL>particleGenIdx = particleState['<STR_LIT>']<EOL>particleId = particleState['<STR_LIT:id>']<EOL>if genIdx is not None and particleGenIdx != genIdx:<EOL><INDENT>continue<EOL><DEDENT>if completed is not None and (completed != isCompleted):<EOL><INDENT>continue<EOL><DEDENT>if matured is not None and (matured != isMatured):<EOL><INDENT>continue<EOL><DEDENT>if lastDescendentand (self._particleLatestGenIdx[particleId] != particleGenIdx):<EOL><INDENT>continue<EOL><DEDENT>particleStates.append(particleState)<EOL>modelIds.append(entry['<STR_LIT>'])<EOL>errScores.append(entry['<STR_LIT>'])<EOL>completedFlags.append(isCompleted)<EOL>maturedFlags.append(isMatured)<EOL><DEDENT>return (particleStates, modelIds, errScores, completedFlags, maturedFlags)<EOL>

Return a list of particleStates for all particles we know about in
        the given swarm, their model Ids, and metric results.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:  A string representation of the sorted list of encoders in this
                     swarm. For example '__address_encoder.__gym_encoder'

        genIdx:  If not None, only return particles at this specific generation
                      index.

        completed:   If not None, only return particles of the given state (either
                    completed if 'completed' is True, or running if 'completed'
                    is false

        matured:   If not None, only return particles of the given state (either
                    matured if 'matured' is True, or not matured if 'matured'
                    is false. Note that any model which has completed is also
                    considered matured.

        lastDescendent: If True, only return particles that are the last descendent,
                    that is, the highest generation index for a given particle Id

        retval:  (particleStates, modelIds, errScores, completed, matured)
                  particleStates: list of particleStates
                  modelIds: list of modelIds
                  errScores: list of errScores, numpy.inf is plugged in
                                  if we don't have a result yet
                  completed: list of completed booleans
                  matured: list of matured booleans


self.logger.debug("<STR_LIT>" %(self._modelOrphanIntervalSecs))<EOL>while True:<EOL><INDENT>orphanedModelId = self._cjDAO.modelAdoptNextOrphan(self._jobID,<EOL>self._modelOrphanIntervalSecs)<EOL>if orphanedModelId is None:<EOL><INDENT>return<EOL><DEDENT>self.logger.info("<STR_LIT>" % (orphanedModelId))<EOL>for attempt in range(<NUM_LIT:100>):<EOL><INDENT>paramsHash = hashlib.md5("<STR_LIT>" % (orphanedModelId,<EOL>attempt)).digest()<EOL>particleHash = hashlib.md5("<STR_LIT>" % (orphanedModelId,<EOL>attempt)).digest()<EOL>try:<EOL><INDENT>self._cjDAO.modelSetFields(orphanedModelId,<EOL>dict(engParamsHash=paramsHash,<EOL>engParticleHash=particleHash))<EOL>success = True<EOL><DEDENT>except:<EOL><INDENT>success = False<EOL><DEDENT>if success:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not success:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._cjDAO.modelSetCompleted(modelID=orphanedModelId,<EOL>completionReason=ClientJobsDAO.CMPL_REASON_ORPHAN,<EOL>completionMsg="<STR_LIT>")<EOL>self._resultsDB.update(modelID=orphanedModelId,<EOL>modelParams=None,<EOL>modelParamsHash=paramsHash,<EOL>metricResult=None,<EOL>completed = True,<EOL>completionReason = ClientJobsDAO.CMPL_REASON_ORPHAN,<EOL>matured = True,<EOL>numRecords = <NUM_LIT:0>)<EOL><DEDENT>

If there are any models that haven't been updated in a while, consider
        them dead, and mark them as hidden in our resultsDB. We also change the
        paramsHash and particleHash of orphaned models so that we can
        re-generate that particle and/or model again if we desire.

        Parameters:
        ----------------------------------------------------------------------
        retval:


<EOL>self._velocity *= <NUM_LIT> / self._inertia<EOL>maxV = (self.max - self.min)/<NUM_LIT:2><EOL>if self._velocity > maxV:<EOL><INDENT>self._velocity = maxV<EOL><DEDENT>elif self._velocity < -maxV:<EOL><INDENT>self._velocity = -maxV<EOL><DEDENT>if self._position == self.max and self._velocity > <NUM_LIT:0>:<EOL><INDENT>self._velocity *= -<NUM_LIT:1><EOL><DEDENT>if self._position == self.min and self._velocity < <NUM_LIT:0>:<EOL><INDENT>self._velocity *= -<NUM_LIT:1><EOL><DEDENT>


return ("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>self.min, self.max, self.stepSize, self.getPosition(),<EOL>self._position, self._velocity, self._bestPosition,<EOL>self._bestResult))<EOL>


def newPosition(self, globalBestPosition, rng):

<EOL>lb=float(Configuration.get("<STR_LIT>"))<EOL>ub=float(Configuration.get("<STR_LIT>"))<EOL>self._velocity = (self._velocity * self._inertia + rng.uniform(lb, ub) *<EOL>self._cogRate * (self._bestPosition - self.getPosition()))<EOL>if globalBestPosition is not None:<EOL><INDENT>self._velocity += rng.uniform(lb, ub) * self._socRate * (<EOL>globalBestPosition - self.getPosition())<EOL><DEDENT>self._position += self._velocity<EOL>self._position = max(self.min, self._position)<EOL>self._position = min(self.max, self._position)<EOL>return self.getPosition()<EOL>


def getDict(self, encoderName, flattenedChosenValues):

encoder = dict(fieldname=self.fieldName,<EOL>name=self.name)<EOL>for encoderArg, value in self.kwArgs.items():<EOL><INDENT>if isinstance(value, PermuteVariable):<EOL><INDENT>value = flattenedChosenValues["<STR_LIT>" % (encoderName, encoderArg)]<EOL><DEDENT>encoder[encoderArg] = value<EOL><DEDENT>if '<STR_LIT:.>' in self.encoderClass:<EOL><INDENT>(encoder['<STR_LIT:type>'], argName) = self.encoderClass.split('<STR_LIT:.>')<EOL>argValue = (encoder['<STR_LIT:w>'], encoder['<STR_LIT>'])<EOL>encoder[argName] = argValue<EOL>encoder.pop('<STR_LIT:w>')<EOL>encoder.pop('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>encoder['<STR_LIT:type>'] = self.encoderClass<EOL><DEDENT>return encoder<EOL>

Return a dict that can be used to construct this encoder. This dict
        can be passed directly to the addMultipleEncoders() method of the
        multi encoder.

        Parameters:
        ----------------------------------------------------------------------
        encoderName:            name of the encoder
        flattenedChosenValues:  dict of the flattened permutation variables. Any
                                  variables within this dict whose key starts
                                  with encoderName will be substituted for
                                  encoder constructor args which are being
                                  permuted over.


raise NotImplementedError<EOL>

Return the current state of this particle. This is used for
        communicating our state into a model record entry so that it can be
        instantiated on another worker.


def pushAwayFrom(self, otherPositions, rng):

<EOL>if self.max == self.min:<EOL><INDENT>return<EOL><DEDENT>numPositions = len(otherPositions) * <NUM_LIT:4><EOL>if numPositions == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>stepSize = float(self.max-self.min) / numPositions<EOL>positions = numpy.arange(self.min, self.max + stepSize, stepSize)<EOL>numPositions = len(positions)<EOL>weights = numpy.zeros(numPositions)<EOL>maxDistanceSq = -<NUM_LIT:1> * (stepSize ** <NUM_LIT:2>)<EOL>for pos in otherPositions:<EOL><INDENT>distances = pos - positions<EOL>varWeights = numpy.exp(numpy.power(distances, <NUM_LIT:2>) / maxDistanceSq)<EOL>weights += varWeights<EOL><DEDENT>positionIdx = weights.argmin()<EOL>self._position = positions[positionIdx]<EOL>self._bestPosition = self.getPosition()<EOL>self._velocity *= rng.choice([<NUM_LIT:1>, -<NUM_LIT:1>])<EOL>


self._positionIdx = self.choices.index(state['<STR_LIT>'])<EOL>self._bestPositionIdx = self.choices.index(state['<STR_LIT>'])<EOL>self._bestResult = state['<STR_LIT>']<EOL>


raise NotImplementedError<EOL>

Set the current state of this particle. This is counterpart to getState.


def __init__(self, min, max, stepSize=None, inertia=None, cogRate=None,<EOL>socRate=None):

super(PermuteFloat, self).__init__()<EOL>self.min = min<EOL>self.max = max<EOL>self.stepSize = stepSize<EOL>self._position = (self.max + self.min) / <NUM_LIT><EOL>self._velocity = (self.max - self.min) / <NUM_LIT><EOL>self._inertia = (float(Configuration.get("<STR_LIT>"))<EOL>if inertia is None else inertia)<EOL>self._cogRate = (float(Configuration.get("<STR_LIT>"))<EOL>if cogRate is None else cogRate)<EOL>self._socRate = (float(Configuration.get("<STR_LIT>"))<EOL>if socRate is None else socRate)<EOL>self._bestPosition = self.getPosition()<EOL>self._bestResult = None<EOL>

Construct a variable that permutes over floating point values using
        the Particle Swarm Optimization (PSO) algorithm. See descriptions of
        PSO (i.e. http://en.wikipedia.org/wiki/Particle_swarm_optimization)
        for references to the inertia, cogRate, and socRate parameters.

        Parameters:
        -----------------------------------------------------------------------
        min:          min allowed value of position
        max:          max allowed value of position
        stepSize:     if not None, the position must be at min + N * stepSize,
                        where N is an integer
        inertia:      The inertia for the particle.
        cogRate:      This parameter controls how much the particle is affected
                        by its distance from it's local best position
        socRate:      This parameter controls how much the particle is affected
                        by its distance from the global best position


<EOL>self.verbosity = <NUM_LIT:0><EOL>self._testValidPositions(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, stepSize=<NUM_LIT:0.5>)<EOL>self._testValidPositions(varClass=PermuteInt, minValue=<NUM_LIT:2>,<EOL>maxValue=<NUM_LIT:11>, stepSize=<NUM_LIT:3>)<EOL>self._testValidPositions(varClass=PermuteInt, minValue=<NUM_LIT:2>,<EOL>maxValue=<NUM_LIT:11>, stepSize=<NUM_LIT:1>)<EOL>self._testConvergence(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteFloat, minValue=<NUM_LIT>,<EOL>maxValue=<NUM_LIT>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteInt, minValue=<NUM_LIT:1>,<EOL>maxValue=<NUM_LIT:20>, targetValue=<NUM_LIT>)<EOL>self._testConvergence(varClass=PermuteInt, minValue=<NUM_LIT:1>,<EOL>maxValue=<NUM_LIT:20>, targetValue=<NUM_LIT:1>)<EOL>self._testChoices()<EOL>


raise NotImplementedError<EOL>

for int vars, returns position to nearest int

        Parameters:
        --------------------------------------------------------------
        retval:     current position


<EOL>globalBestPosition = None<EOL>if self._hsObj._speculativeParticles:<EOL><INDENT>genIdx = self.genIdx<EOL><DEDENT>else:<EOL><INDENT>genIdx = self.genIdx - <NUM_LIT:1><EOL><DEDENT>if genIdx >= <NUM_LIT:0>:<EOL><INDENT>(bestModelId, _) = self._resultsDB.bestModelIdAndErrScore(self.swarmId,<EOL>genIdx)<EOL>if bestModelId is not None:<EOL><INDENT>(particleState, _, _, _, _) = self._resultsDB.getParticleInfo(<EOL>bestModelId)<EOL>globalBestPosition = Particle.getPositionFromState(particleState)<EOL><DEDENT><DEDENT>for (varName, var) in self.permuteVars.iteritems():<EOL><INDENT>if whichVars is not None and varName not in whichVars:<EOL><INDENT>continue<EOL><DEDENT>if globalBestPosition is None:<EOL><INDENT>var.newPosition(None, self._rng)<EOL><DEDENT>else:<EOL><INDENT>var.newPosition(globalBestPosition[varName], self._rng)<EOL><DEDENT><DEDENT>position = self.getPosition()<EOL>if self.logger.getEffectiveLevel() <= logging.DEBUG:<EOL><INDENT>msg = StringIO.StringIO()<EOL>print >> msg, "<STR_LIT>" % (pprint.pformat(position,<EOL>indent=<NUM_LIT:4>))<EOL>print >> msg, "<STR_LIT>"<EOL>for (varName, var) in self.permuteVars.iteritems():<EOL><INDENT>print >> msg, "<STR_LIT>" % (varName, str(var))<EOL><DEDENT>self.logger.debug(msg.getvalue())<EOL>msg.close()<EOL><DEDENT>return position<EOL>

Choose a new position based on results obtained so far from all other
        particles.

        Parameters:
        --------------------------------------------------------------
        whichVars:       If not None, only move these variables
        retval:               new position


def initStateFrom(self, particleId, particleState, newBest):

<EOL>if newBest:<EOL><INDENT>(bestResult, bestPosition) = self._resultsDB.getParticleBest(particleId)<EOL><DEDENT>else:<EOL><INDENT>bestResult = bestPosition = None<EOL><DEDENT>varStates = particleState['<STR_LIT>']<EOL>for varName in varStates.keys():<EOL><INDENT>varState = copy.deepcopy(varStates[varName])<EOL>if newBest:<EOL><INDENT>varState['<STR_LIT>'] = bestResult<EOL><DEDENT>if bestPosition is not None:<EOL><INDENT>varState['<STR_LIT>'] = bestPosition[varName]<EOL><DEDENT>self.permuteVars[varName].setState(varState)<EOL><DEDENT>

Init all of our variable positions, velocities, and optionally the best
        result and best position from the given particle.

        If newBest is true, we get the best result and position for this new
        generation from the resultsDB, This is used when evoloving a particle
        because the bestResult and position as stored in was the best AT THE TIME
        THAT PARTICLE STARTED TO RUN and does not include the best since that
        particle completed.


result = dict()<EOL>for (varName, value) in self.permuteVars.iteritems():<EOL><INDENT>result[varName] = value.getPosition()<EOL><DEDENT>return result<EOL>

Return the position of this particle. This returns a dict() of key
        value pairs where each key is the name of the flattened permutation
        variable and the value is its chosen value.

        Parameters:
        --------------------------------------------------------------
        retval:     dict() of flattened permutation choices


for (varName, var) in self.permuteVars.iteritems():<EOL><INDENT>var.agitate()<EOL><DEDENT>self.newPosition()<EOL>

Agitate this particle so that it is likely to go to a new position.
        Every time agitate is called, the particle is jiggled an even greater
        amount.

        Parameters:
        --------------------------------------------------------------
        retval:               None


sprintInfo = self._state['<STR_LIT>'][sprintIdx]<EOL>return (sprintInfo['<STR_LIT>'],<EOL>sprintInfo['<STR_LIT>'])<EOL>

Return the best model ID and it's errScore from the given sprint.
        If the sprint has not completed yet, the bestModelID will be None.

        Parameters:
        ---------------------------------------------------------------------
        retval:   (modelId, errScore)


<EOL>self._hsObj = hsObj<EOL>self.logger = self._hsObj.logger<EOL>self._state = None<EOL>self._priorStateJSON = None<EOL>self._dirty = False<EOL>self.readStateFromDB()<EOL>

Create our state object.

        Parameters:
        ---------------------------------------------------------------------
        hsObj:     Reference to the HypersesarchV2 instance
        cjDAO:     ClientJobsDAO instance
        logger:    logger to use
        jobID:     our JobID


<EOL>if not self._dirty:<EOL><INDENT>return True<EOL><DEDENT>self._state['<STR_LIT>'] = time.time()<EOL>newStateJSON = json.dumps(self._state)<EOL>success = self._hsObj._cjDAO.jobSetFieldIfEqual(self._hsObj._jobID,<EOL>'<STR_LIT>', str(newStateJSON), str(self._priorStateJSON))<EOL>if success:<EOL><INDENT>self.logger.debug("<STR_LIT>" %(pprint.pformat(self._state, indent=<NUM_LIT:4>)))<EOL>self._priorStateJSON = newStateJSON<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug("<STR_LIT>" %(pprint.pformat(self._state, indent=<NUM_LIT:4>)))<EOL>self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(self._hsObj._jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>self._state =  json.loads(self._priorStateJSON)<EOL>self.logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % (pprint.pformat(self._state, indent=<NUM_LIT:4>)))<EOL><DEDENT>return success<EOL>

Update the state in the job record with our local changes (if any).
        If we don't have the latest state in our priorStateJSON, then re-load
        in the latest state and return False. If we were successful writing out
        our changes, return True

        Parameters:
        ---------------------------------------------------------------------
        retval:    True if we were successful writing out our changes
                   False if our priorState is not the latest that was in the DB.
                   In this case, we will re-load our state from the DB


def setSwarmState(self, swarmId, newStatus):

assert (newStatus in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>swarmInfo = self._state['<STR_LIT>'][swarmId]<EOL>if swarmInfo['<STR_LIT:status>'] == newStatus:<EOL><INDENT>return<EOL><DEDENT>if swarmInfo['<STR_LIT:status>'] == '<STR_LIT>' and newStatus == '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>self._dirty = True<EOL>swarmInfo['<STR_LIT:status>'] = newStatus<EOL>if newStatus == '<STR_LIT>':<EOL><INDENT>(modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)<EOL>swarmInfo['<STR_LIT>'] = modelId<EOL>swarmInfo['<STR_LIT>'] = errScore<EOL><DEDENT>if newStatus != '<STR_LIT>' and swarmId in self._state['<STR_LIT>']:<EOL><INDENT>self._state['<STR_LIT>'].remove(swarmId)<EOL><DEDENT>if newStatus=='<STR_LIT>':<EOL><INDENT>self._hsObj.killSwarmParticles(swarmId)<EOL><DEDENT>sprintIdx = swarmInfo['<STR_LIT>']<EOL>self.isSprintActive(sprintIdx)<EOL>sprintInfo = self._state['<STR_LIT>'][sprintIdx]<EOL>statusCounts = dict(active=<NUM_LIT:0>, completing=<NUM_LIT:0>, completed=<NUM_LIT:0>, killed=<NUM_LIT:0>)<EOL>bestModelIds = []<EOL>bestErrScores = []<EOL>for info in self._state['<STR_LIT>'].values():<EOL><INDENT>if info['<STR_LIT>'] != sprintIdx:<EOL><INDENT>continue<EOL><DEDENT>statusCounts[info['<STR_LIT:status>']] += <NUM_LIT:1><EOL>if info['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>bestModelIds.append(info['<STR_LIT>'])<EOL>bestErrScores.append(info['<STR_LIT>'])<EOL><DEDENT><DEDENT>if statusCounts['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>sprintStatus = '<STR_LIT>'<EOL><DEDENT>elif statusCounts['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>sprintStatus = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>sprintStatus = '<STR_LIT>'<EOL><DEDENT>sprintInfo['<STR_LIT:status>'] = sprintStatus<EOL>if sprintStatus == '<STR_LIT>':<EOL><INDENT>if len(bestErrScores) > <NUM_LIT:0>:<EOL><INDENT>whichIdx = numpy.array(bestErrScores).argmin()<EOL>sprintInfo['<STR_LIT>'] = bestModelIds[whichIdx]<EOL>sprintInfo['<STR_LIT>'] = bestErrScores[whichIdx]<EOL><DEDENT>else:<EOL><INDENT>sprintInfo['<STR_LIT>'] = <NUM_LIT:0><EOL>sprintInfo['<STR_LIT>'] = numpy.inf<EOL><DEDENT>bestPrior = numpy.inf<EOL>for idx in range(sprintIdx):<EOL><INDENT>if self._state['<STR_LIT>'][idx]['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>(_, errScore) = self.bestModelInCompletedSprint(idx)<EOL>if errScore is None:<EOL><INDENT>errScore = numpy.inf<EOL><DEDENT><DEDENT>else:<EOL><INDENT>errScore = numpy.inf<EOL><DEDENT>if errScore < bestPrior:<EOL><INDENT>bestPrior = errScore<EOL><DEDENT><DEDENT>if sprintInfo['<STR_LIT>'] >= bestPrior:<EOL><INDENT>self._state['<STR_LIT>'] = sprintIdx-<NUM_LIT:1><EOL><DEDENT>if self._state['<STR_LIT>'] is not Noneand not self.anyGoodSprintsActive():<EOL><INDENT>self._state['<STR_LIT>'] = True<EOL><DEDENT><DEDENT>

Change the given swarm's state to 'newState'. If 'newState' is
        'completed', then bestModelId and bestErrScore must be provided.

        Parameters:
        ---------------------------------------------------------------------
        swarmId:      swarm Id
        newStatus:    new status, either 'active', 'completing', 'completed', or
                        'killed'


<EOL>numExistingSprints = len(self._state['<STR_LIT>'])<EOL>if self._hsObj._searchType == HsSearchType.legacyTemporal:<EOL><INDENT>if numExistingSprints <= <NUM_LIT:2>:<EOL><INDENT>return<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if numExistingSprints <= <NUM_LIT:1>:<EOL><INDENT>return<EOL><DEDENT><DEDENT>completedSwarms = self.getCompletedSwarms()<EOL>completedSwarms = [(swarm, self._state["<STR_LIT>"][swarm],<EOL>self._state["<STR_LIT>"][swarm]["<STR_LIT>"])for swarm in completedSwarms]<EOL>completedMatrix = [[] for i in range(numExistingSprints)]<EOL>for swarm in completedSwarms:<EOL><INDENT>completedMatrix[swarm[<NUM_LIT:1>]["<STR_LIT>"]].append(swarm)<EOL><DEDENT>for sprint in completedMatrix:<EOL><INDENT>sprint.sort(key=itemgetter(<NUM_LIT:2>))<EOL><DEDENT>activeSwarms = self.getActiveSwarms()<EOL>activeSwarms.extend(self.getCompletingSwarms())<EOL>activeSwarms = [(swarm, self._state["<STR_LIT>"][swarm],<EOL>self._state["<STR_LIT>"][swarm]["<STR_LIT>"])for swarm in activeSwarms]<EOL>activeMatrix = [[] for i in range(numExistingSprints)]<EOL>for swarm in activeSwarms:<EOL><INDENT>activeMatrix[swarm[<NUM_LIT:1>]["<STR_LIT>"]].append(swarm)<EOL><DEDENT>for sprint in activeMatrix:<EOL><INDENT>sprint.sort(key=itemgetter(<NUM_LIT:2>))<EOL><DEDENT>toKill = []<EOL>for i in range(<NUM_LIT:1>, numExistingSprints):<EOL><INDENT>for swarm in activeMatrix[i]:<EOL><INDENT>curSwarmEncoders = swarm[<NUM_LIT:0>].split("<STR_LIT:.>")<EOL>if(len(activeMatrix[i-<NUM_LIT:1>])==<NUM_LIT:0>):<EOL><INDENT>if i==<NUM_LIT:2> and (self._hsObj._tryAll3FieldCombinations orself._hsObj._tryAll3FieldCombinationsWTimestamps):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>bestInPrevious = completedMatrix[i-<NUM_LIT:1>][<NUM_LIT:0>]<EOL>bestEncoders = bestInPrevious[<NUM_LIT:0>].split('<STR_LIT:.>')<EOL>for encoder in bestEncoders:<EOL><INDENT>if not encoder in curSwarmEncoders:<EOL><INDENT>toKill.append(swarm)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if len(toKill) > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" + str(toKill))<EOL><DEDENT>for swarm in toKill:<EOL><INDENT>self.setSwarmState(swarm[<NUM_LIT:0>], "<STR_LIT>")<EOL><DEDENT>return<EOL>

See if we can kill off some speculative swarms. If an earlier sprint
        has finally completed, we can now tell which fields should *really* be present
        in the sprints we've already started due to speculation, and kill off the
        swarms that should not have been included.


<EOL>if self._hsObj._fixedFields is not None:<EOL><INDENT>return dict(), dict()<EOL><DEDENT>predictedEncoderName = self._hsObj._predictedFieldEncoder<EOL>fieldScores = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>encodersUsed = swarmId.split('<STR_LIT:.>')<EOL>if len(encodersUsed) != <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>field = self.getEncoderNameFromKey(encodersUsed[<NUM_LIT:0>])<EOL>bestScore = info['<STR_LIT>']<EOL>if bestScore is None:<EOL><INDENT>(_modelId, bestScore) =self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)<EOL><DEDENT>fieldScores.append((bestScore, field))<EOL><DEDENT>if self._hsObj._searchType == HsSearchType.legacyTemporal:<EOL><INDENT>assert(len(fieldScores)==<NUM_LIT:1>)<EOL>(baseErrScore, baseField) = fieldScores[<NUM_LIT:0>]<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>encodersUsed = swarmId.split('<STR_LIT:.>')<EOL>if len(encodersUsed) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>fields = [self.getEncoderNameFromKey(name) for name in encodersUsed]<EOL>fields.remove(baseField)<EOL>fieldScores.append((info['<STR_LIT>'], fields[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fieldScores.sort(reverse=True)<EOL>if self._hsObj._maxBranching > <NUM_LIT:0>and len(fieldScores) > self._hsObj._maxBranching:<EOL><INDENT>baseErrScore = fieldScores[-self._hsObj._maxBranching-<NUM_LIT:1>][<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>baseErrScore = fieldScores[<NUM_LIT:0>][<NUM_LIT:0>]<EOL><DEDENT><DEDENT>pctFieldContributionsDict = dict()<EOL>absFieldContributionsDict = dict()<EOL>if baseErrScore is not None:<EOL><INDENT>if abs(baseErrScore) < <NUM_LIT>:<EOL><INDENT>baseErrScore = <NUM_LIT><EOL><DEDENT>for (errScore, field) in fieldScores:<EOL><INDENT>if errScore is not None:<EOL><INDENT>pctBetter = (baseErrScore - errScore) * <NUM_LIT> / baseErrScore<EOL><DEDENT>else:<EOL><INDENT>pctBetter = <NUM_LIT:0.0><EOL>errScore = baseErrScore   <EOL><DEDENT>pctFieldContributionsDict[field] = pctBetter<EOL>absFieldContributionsDict[field] = baseErrScore - errScore<EOL><DEDENT><DEDENT>self.logger.debug("<STR_LIT>" % (pctFieldContributionsDict))<EOL>return pctFieldContributionsDict, absFieldContributionsDict<EOL>

Return the field contributions statistics.

        Parameters:
        ---------------------------------------------------------------------
        retval:   Dictionary where the keys are the field names and the values
                    are how much each field contributed to the best score.


swarmIds = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>if info['<STR_LIT>'] == sprintIdx and info['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>swarmIds.append(swarmId)<EOL><DEDENT><DEDENT>return swarmIds<EOL>

Return the list of swarms in the given sprint that were not killed.
        This is called when we are trying to figure out which encoders to carry
        forward to the next sprint. We don't want to carry forward encoder
        combintations which were obviously bad (in killed swarms).

        Parameters:
        ---------------------------------------------------------------------
        retval:   list of active swarm Ids in the given sprint


numExistingSprints = len(self._state['<STR_LIT>'])<EOL>if sprintIdx >= numExistingSprints:<EOL><INDENT>return False<EOL><DEDENT>return (self._state['<STR_LIT>'][sprintIdx]['<STR_LIT:status>'] == '<STR_LIT>')<EOL>

Return True if the given sprint has completed.


<EOL>swarms = self.getAllSwarms(sprintIdx)<EOL>bestModelId = None<EOL>bestErrScore = numpy.inf<EOL>for swarmId in swarms:<EOL><INDENT>(modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)<EOL>if errScore < bestErrScore:<EOL><INDENT>bestModelId = modelId<EOL>bestErrScore = errScore<EOL><DEDENT><DEDENT>return (bestModelId, bestErrScore)<EOL>

Return the best model ID and it's errScore from the given sprint,
        which may still be in progress. This returns the best score from all models
        in the sprint which have matured so far.

        Parameters:
        ---------------------------------------------------------------------
        retval:   (modelId, errScore)


swarmIds = []<EOL>for swarmId, info in self._state['<STR_LIT>'].items():<EOL><INDENT>if info['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>swarmIds.append(swarmId)<EOL><DEDENT><DEDENT>return swarmIds<EOL>


def critical(self, msg, *args, **kwargs):

self._baseLogger.critical(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'CRITICAL'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.critical("Houston, we have a %s", "major disaster", exc_info=1)


self._baseLogger.debug(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'DEBUG'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)


self._baseLogger.error(self, self.getExtendedMsg(msg), *args, **kwargs)<EOL>

Log 'msg % args' with severity 'ERROR'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.error("Houston, we have a %s", "major problem", exc_info=1)


def log(self, level, msg, *args, **kwargs):

self._baseLogger.log(self, level, self.getExtendedMsg(msg), *args,<EOL>**kwargs)<EOL>

Log 'msg % args' with the integer severity 'level'.

To pass exception information, use the keyword argument exc_info with
a true value, e.g.

logger.log(level, "We have a %s", "mysterious problem", exc_info=1)


if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>envValue = os.environ.get("<STR_LIT>" % (cls.envPropPrefix,<EOL>prop.replace('<STR_LIT:.>', '<STR_LIT:_>')), None)<EOL>if envValue is not None:<EOL><INDENT>return envValue<EOL><DEDENT>return cls._properties[prop]<EOL>

Retrieve the requested property as a string. If property does not exist,
        then KeyError will be raised.

        Parameters:
        ----------------------------------------------------------------
        prop:        name of the property
        retval:      property value as a string


configPaths = []<EOL>if cls._configPaths is not None:<EOL><INDENT>return cls._configPaths<EOL><DEDENT>else:<EOL><INDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>configVar = os.environ['<STR_LIT>']<EOL>configPaths = configVar.split(os.pathsep)<EOL><DEDENT>return configPaths<EOL><DEDENT>

Return the list of paths to search for configuration files.

        Parameters:
        ----------------------------------------------------------------
        retval:    list of paths.


if cls._properties is None:<EOL><INDENT>cls._readStdConfigFiles()<EOL><DEDENT>result = dict(cls._properties)<EOL>keys = list(os.environ.keys())<EOL>replaceKeys = [x for x in keys if x.startswith(cls.envPropPrefix)]<EOL>for envKey in replaceKeys:<EOL><INDENT>key = envKey[len(cls.envPropPrefix):]<EOL>key = key.replace('<STR_LIT:_>', '<STR_LIT:.>')<EOL>result[key] = os.environ[envKey]<EOL><DEDENT>return result<EOL>

Return a dict containing all of the configuration properties

        Parameters:
        ----------------------------------------------------------------
        retval:      dict containing all configuration properties.


cls._properties = None<EOL>cls._configPaths = None<EOL>


paths = cls.getConfigPaths()<EOL>for p in paths:<EOL><INDENT>testPath = os.path.join(p, filename)<EOL>if os.path.isfile(testPath):<EOL><INDENT>return os.path.join(p, filename)<EOL><DEDENT><DEDENT>

Search the configuration path (specified via the NTA_CONF_PATH
        environment variable) for the given filename. If found, return the complete
        path to the file.

        Parameters:
        ----------------------------------------------------------------
        filename:  name of file to locate


@classmethod<EOL><INDENT>def setCustomProperty(cls, propertyName, value):<DEDENT>

cls.setCustomProperties({propertyName: value})<EOL>

Set a single custom setting and persist it to the custom
        configuration store.

        Parameters:
        ----------------------------------------------------------------
        propertyName: string containing the name of the property to get
        value: value to set the property to


_getLogger().info("<STR_LIT>",<EOL>properties, traceback.format_stack())<EOL>_CustomConfigurationFileWrapper.edit(properties)<EOL>for propertyName, value in properties.items():<EOL><INDENT>cls.set(propertyName, value)<EOL><DEDENT>

Set multiple custom properties and persist them to the custom
        configuration store.

        Parameters:
        ----------------------------------------------------------------
        properties: a dict of property name/value pairs to set


<EOL>cls.readConfigFile(DEFAULT_CONFIG)<EOL>cls.readConfigFile(USER_CONFIG)<EOL>

Read in all standard configuration files


cls.readConfigFile(_CustomConfigurationFileWrapper.customFileName)<EOL>

Loads custom configuration settings from their persistent storage.
        DO NOT CALL THIS: It's typically not necessary to call this method
        directly - see NOTE below.

        NOTE: this method exists *solely* for the benefit of prepare_conf.py, which
        needs to load configuration files selectively.


if cls._path is None:<EOL><INDENT>cls._setPath()<EOL><DEDENT>return cls._path<EOL>

Get the path of the custom configuration file


if not os.path.exists(cls.getPath()):<EOL><INDENT>return dict()<EOL><DEDENT>properties = Configuration._readConfigFile(os.path.basename(<EOL>cls.getPath()), os.path.dirname(cls.getPath()))<EOL>values = dict()<EOL>for propName in properties:<EOL><INDENT>if '<STR_LIT:value>' in properties[propName]:<EOL><INDENT>values[propName] = properties[propName]['<STR_LIT:value>']<EOL><DEDENT><DEDENT>return values<EOL>

Returns a dict of all temporary values in custom configuration file


<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=False)<EOL>

Clear all configuration properties from in-memory cache, but do NOT
        alter the custom configuration file. Used in unit-testing.


self._logger.debug("<STR_LIT>" % (self._modelID))<EOL>periodic = self._initPeriodicActivities()<EOL>self._optimizedMetricLabel = self._optimizeKeyPattern<EOL>self._reportMetricLabels = [self._optimizeKeyPattern]<EOL>if self._iterations >= <NUM_LIT:0>:<EOL><INDENT>iterTracker = iter(xrange(self._iterations))<EOL><DEDENT>else:<EOL><INDENT>iterTracker = iter(itertools.count())<EOL><DEDENT>doSysExit = False<EOL>if self._sysExitModelRange is not None:<EOL><INDENT>modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)<EOL>modelIDs = [x[<NUM_LIT:0>] for x in modelAndCounters]<EOL>modelIDs.sort()<EOL>(beg,end) = self._sysExitModelRange<EOL>if self._modelID in modelIDs[int(beg):int(end)]:<EOL><INDENT>doSysExit = True<EOL><DEDENT><DEDENT>if self._delayModelRange is not None:<EOL><INDENT>modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)<EOL>modelIDs = [x[<NUM_LIT:0>] for x in modelAndCounters]<EOL>modelIDs.sort()<EOL>(beg,end) = self._delayModelRange<EOL>if self._modelID in modelIDs[int(beg):int(end)]:<EOL><INDENT>time.sleep(<NUM_LIT:10>)<EOL><DEDENT><DEDENT>if self._errModelRange is not None:<EOL><INDENT>modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)<EOL>modelIDs = [x[<NUM_LIT:0>] for x in modelAndCounters]<EOL>modelIDs.sort()<EOL>(beg,end) = self._errModelRange<EOL>if self._modelID in modelIDs[int(beg):int(end)]:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>if self._delay is not None:<EOL><INDENT>time.sleep(self._delay)<EOL><DEDENT>self._currentRecordIndex = <NUM_LIT:0><EOL>while True:<EOL><INDENT>if self._isKilled:<EOL><INDENT>break<EOL><DEDENT>if self._isCanceled:<EOL><INDENT>break<EOL><DEDENT>if self._isMature:<EOL><INDENT>if not self._isBestModel:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED<EOL>break<EOL><DEDENT>else:<EOL><INDENT>self._cmpReason = self._jobsDAO.CMPL_REASON_EOF<EOL><DEDENT><DEDENT>try:<EOL><INDENT>self._currentRecordIndex = next(iterTracker)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>self._writePrediction(ModelResult(None, None, None, None))<EOL>periodic.tick()<EOL>if self.__shouldSysExit(self._currentRecordIndex):<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if self._busyWaitTime is not None:<EOL><INDENT>time.sleep(self._busyWaitTime)<EOL>self.__computeWaitTime()<EOL><DEDENT>if doSysExit:<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if self._jobFailErr:<EOL><INDENT>raise utils.JobFailException("<STR_LIT>",<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>if self._doFinalize:<EOL><INDENT>if not self._makeCheckpoint:<EOL><INDENT>self._model = None<EOL><DEDENT>if self._finalDelay is not None:<EOL><INDENT>time.sleep(self._finalDelay)<EOL><DEDENT>self._finalize()<EOL><DEDENT>self._logger.info("<STR_LIT>"% (self._modelID))<EOL>return (self._cmpReason, None)<EOL>

Runs the given OPF task against the given Model instance


<EOL>if self._params['<STR_LIT>'] is not Noneand self._params['<STR_LIT>'] is not None:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if self._sleepModelRange is not None:<EOL><INDENT>range, delay = self._sleepModelRange.split('<STR_LIT::>')<EOL>delay = float(delay)<EOL>range = map(int, range.split('<STR_LIT:U+002C>'))<EOL>modelIDs = self._jobsDAO.jobGetModelIDs(self._jobID)<EOL>modelIDs.sort()<EOL>range[<NUM_LIT:1>] = min(range[<NUM_LIT:1>], len(modelIDs))<EOL>if self._modelID in modelIDs[range[<NUM_LIT:0>]:range[<NUM_LIT:1>]]:<EOL><INDENT>self._delay = delay<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._delay = self._params['<STR_LIT>']<EOL><DEDENT>

Computes the amount of time (if any) to delay the run of this model.
        This can be determined by two mutually exclusive parameters:
        delay and sleepModelRange.

        'delay' specifies the number of seconds a model should be delayed. If a list
        is specified, the appropriate amount of delay is determined by using the
        model's modelIndex property.

        However, this doesn't work when testing orphaned models, because the
        modelIndex will be the same for every recovery attempt. Therefore, every
        recovery attempt will also be delayed and potentially orphaned.

        'sleepModelRange' doesn't use the modelIndex property for a model, but rather
        sees which order the model is in the database, and uses that to determine
        whether or not a model should be delayed.


super(OPFDummyModelRunner, self).__init__(modelID=modelID,<EOL>jobID=jobID,<EOL>predictedField=predictedField,<EOL>experimentDir=None,<EOL>reportKeyPatterns=reportKeyPatterns,<EOL>optimizeKeyPattern=optimizeKeyPattern,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=None)<EOL>self._predictionCacheMaxRecords = predictionCacheMaxRecords<EOL>self._streamDef = copy.deepcopy(self._DUMMY_STREAMDEF)<EOL>self._params = copy.deepcopy(self._DEFAULT_PARAMS)<EOL>if '<STR_LIT>' in paramsand '<STR_LIT>' in params['<STR_LIT>']:<EOL><INDENT>self.modelIndex=params['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>self.modelIndex = OPFDummyModelRunner.modelIndex<EOL>OPFDummyModelRunner.modelIndex += <NUM_LIT:1><EOL><DEDENT>self._loadDummyModelParameters(params)<EOL>self._logger.debug("<STR_LIT>", self._params)<EOL>self._busyWaitTime = self._params['<STR_LIT>']<EOL>self._iterations = self._params['<STR_LIT>']<EOL>self._doFinalize = self._params['<STR_LIT>']<EOL>self._delay = self._params['<STR_LIT>']<EOL>self._sleepModelRange = self._params['<STR_LIT>']<EOL>self._makeCheckpoint = self._params['<STR_LIT>']<EOL>self._finalDelay = self._params['<STR_LIT>']<EOL>self._exitAfter = self._params['<STR_LIT>']<EOL>self.randomizeWait = self._params['<STR_LIT>']<EOL>if self._busyWaitTime is not None:<EOL><INDENT>self.__computeWaitTime()<EOL><DEDENT>if self._params['<STR_LIT>'] is not Noneand self._params['<STR_LIT>'] is not None:<EOL><INDENT>raise RuntimeError("<STR_LIT>""<STR_LIT>")<EOL><DEDENT>self.metrics = None<EOL>self.metricValue = None<EOL>if self._params['<STR_LIT>'] is not None:<EOL><INDENT>self.metrics = eval(self._params['<STR_LIT>'])<EOL><DEDENT>elif self._params['<STR_LIT>'] is not None:<EOL><INDENT>self.metricValue = float(self._params['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>self.metrics = OPFDummyModelRunner.metrics[<NUM_LIT:0>]<EOL><DEDENT>if self._params['<STR_LIT>'] is not None:<EOL><INDENT>self._model = self.__createModel(self._params['<STR_LIT>'])<EOL>self.__fieldInfo = self._model.getFieldInfo()<EOL><DEDENT>self._sysExitModelRange = self._params['<STR_LIT>']<EOL>if self._sysExitModelRange is not None:<EOL><INDENT>self._sysExitModelRange = [int(x) for x in self._sysExitModelRange.split('<STR_LIT:U+002C>')]<EOL><DEDENT>self._delayModelRange = self._params['<STR_LIT>']<EOL>if self._delayModelRange is not None:<EOL><INDENT>self._delayModelRange = [int(x) for x in self._delayModelRange.split('<STR_LIT:U+002C>')]<EOL><DEDENT>self._errModelRange = self._params['<STR_LIT>']<EOL>if self._errModelRange is not None:<EOL><INDENT>self._errModelRange = [int(x) for x in self._errModelRange.split('<STR_LIT:U+002C>')]<EOL><DEDENT>self._computModelDelay()<EOL>self._jobFailErr = self._params['<STR_LIT>']<EOL>self._logger.debug("<STR_LIT>", self._modelID, self._params)<EOL>

Parameters:
-------------------------------------------------------------------------
modelID:    ID of this model in the models table

jobID:

params:     a dictionary of parameters for this dummy model. The
            possible keys are:

              delay:          OPTIONAL-This specifies the amount of time
                              (in seconds) that the experiment should wait
                              before STARTING to process records. This is
                              useful for simulating workers that start/end
                              at different times

              finalDelay:     OPTIONAL-This specifies the amount of time
                              (in seconds) that the experiment should wait
                              before it conducts its finalization operations.
                              These operations include checking if the model
                              is the best model, and writing out checkpoints.

              waitTime:       OPTIONAL-The amount of time (in seconds)
                              to wait in a busy loop to simulate
                              computation time on EACH ITERATION

              randomizeWait:  OPTIONAL-([0.0-1.0] ). Default:None
                              If set to a value, the above specified
                              wait time will be randomly be dithered by
                              +/- <randomizeWait>% of the specfied value.
                              For example, if randomizeWait=0.2, the wait
                              time will be dithered by +/- 20% of its value.

              iterations:     OPTIONAL-How many iterations to run the model
                              for. -1 means run forever (default=1)

              metricFunctions: OPTIONAL-A list of single argument functions
                               serialized as strings, which return the metric
                               value given the record number.

                               Mutually exclusive with metricValue

              metricValue:    OPTIONAL-A single value to use for the metric
                              value (used to debug hypersearch).

                              Mutually exclusive with metricFunctions

              finalize:       OPTIONAL-(True/False). Default:True
                              When False, this will prevent the model from
                              recording it's metrics and performing other
                              functions that it usually performs after the
                              model has finished running

              permutationParams: A dict containing the instances of all the
                                  variables being permuted over

              experimentDirectory: REQUIRED-An absolute path to a directory
                                   with a valid description.py file.

                                   NOTE: This does not actually affect the
                                   running of the model or the metrics
                                   produced. It is required to create certain
                                   objects (such as the output stream)

              makeCheckpoint:     True to actually write a checkpoint out to
                                  disk (default: False)

              sysExitModelRange: A string containing two integers 'firstIdx,
                              endIdx'. When present, if we are running the
                              firstIdx'th model up to but not including the
                              endIdx'th model, then do a sys.exit() while
                              running the model. This causes the worker to
                              exit, simulating an orphaned model.

              delayModelRange: A string containing two integers 'firstIdx,
                              endIdx'. When present, if we are running the
                              firstIdx'th model up to but not including the
                              endIdx'th model, then do a delay of 10 sec.
                              while running the model. This causes the
                              worker to run slower and for some other worker
                              to think the model should be orphaned.

              exitAfter:      The number of iterations after which the model
                              should perform a sys exit. This is an
                              alternative way of creating an orphaned model
                              that use's the dummmy model's modelIndex
                              instead of the modelID

              errModelRange: A string containing two integers 'firstIdx,
                              endIdx'. When present, if we are running the
                              firstIdx'th model up to but not including the
                              endIdx'th model, then raise an exception while
                              running the model. This causes the model to
                              fail with a CMPL_REASON_ERROR reason

              sleepModelRange: A string containing 3 integers 'firstIdx,
                              endIdx: delay'. When present, if we are running
                              the firstIdx'th model up to but not including
                              the endIdx'th model, then sleep for delay
                              seconds at the beginning of the run.

              jobFailErr: If true, model will raise a JobFailException
                          which should cause the job to be marked as
                          failed and immediately cancel all other workers.

predictedField:     Name of the input field for which this model is being
                    optimized

reportKeyPatterns:  list of items from the results dict to include in
                    the report. These can be regular expressions.
optimizeKeyPattern: Which report item, if any, we will be optimizing for.
                    This can also be a regular expression, but is an error
                    if it matches more than one key from the experiment's
                    results.

jobsDAO:            Jobs data access object - the interface to the
                    jobs database which has the model's table.

modelCheckpointGUID:
                    A persistent, globally-unique identifier for
                    constructing the model checkpoint key

logLevel:           override logging level to this value, if not None
predictionCacheMaxRecords:
                    Maximum number of records for the prediction output cache.
                    Pass None for the default value.


class DummyLogger:<EOL><INDENT>def writeRecord(self, record): pass<EOL>def writeRecords(self, records, progressCB): pass<EOL>def close(self): pass<EOL><DEDENT>self._predictionLogger = DummyLogger()<EOL>

Creates the model's PredictionLogger object, which is an interface to write
model results to a permanent storage location


self._logger.info(<EOL>"<STR_LIT>",<EOL>self._modelID, self._currentRecordIndex + <NUM_LIT:1>)<EOL>self._updateModelDBResults()<EOL>if not self._isKilled:<EOL><INDENT>self.__updateJobResults()<EOL><DEDENT>else:<EOL><INDENT>self.__deleteOutputCache(self._modelID)<EOL><DEDENT>if self._predictionLogger:<EOL><INDENT>self._predictionLogger.close()<EOL><DEDENT>if self._inputSource: <EOL><INDENT>self._inputSource.close()<EOL><DEDENT>

Run final activities after a model has run. These include recording and
        logging the final score


<EOL>if modelID == self._modelID and self._predictionLogger is not None:<EOL><INDENT>self._predictionLogger.close()<EOL>del self.__predictionCache<EOL>self._predictionLogger = None<EOL>self.__predictionCache = None<EOL><DEDENT>

Delete's the output cache associated with the given modelID. This actually
clears up the resources associated with the cache, rather than deleting al
the records in the cache

Parameters:
-----------------------------------------------------------------------
modelID:      The id of the model whose output cache is being deleted


if self._model is None or self._modelCheckpointGUID is None:<EOL><INDENT>return<EOL><DEDENT>if self._predictionLogger is None:<EOL><INDENT>self._createPredictionLogger()<EOL><DEDENT>predictions = io.StringIO()<EOL>self._predictionLogger.checkpoint(<EOL>checkpointSink=predictions,<EOL>maxRows=int(Configuration.get('<STR_LIT>')))<EOL>self._model.save(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))<EOL>self._jobsDAO.modelSetFields(modelID,<EOL>{'<STR_LIT>':str(self._modelCheckpointGUID)},<EOL>ignoreUnchanged=True)<EOL>self._logger.info("<STR_LIT>"<EOL>"<STR_LIT>", self._modelID, checkpointID)<EOL>return<EOL>

Create a checkpoint from the current model, and store it in a dir named
        after checkpoint GUID, and finally store the GUID in the Models DB


if self._isBestModelStored and not self._isBestModel:<EOL><INDENT>return<EOL><DEDENT>while True:<EOL><INDENT>jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobResultsStr is None:<EOL><INDENT>jobResults = {}<EOL><DEDENT>else:<EOL><INDENT>self._isBestModelStored = True<EOL>if not self._isBestModel:<EOL><INDENT>return<EOL><DEDENT>jobResults = json.loads(jobResultsStr)<EOL><DEDENT>bestModel = jobResults.get('<STR_LIT>', None)<EOL>bestMetric = jobResults.get('<STR_LIT>', None)<EOL>isSaved = jobResults.get('<STR_LIT>', False)<EOL>if (bestModel is not None) and (self._modelID != bestModel):<EOL><INDENT>self._isBestModel = False<EOL>return<EOL><DEDENT>self.__flushPredictionCache()<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>metrics = self._getMetrics()<EOL>jobResults['<STR_LIT>'] = self._modelID<EOL>jobResults['<STR_LIT>'] = metrics[self._optimizedMetricLabel]<EOL>jobResults['<STR_LIT>'] = metrics<EOL>jobResults['<STR_LIT>'] = False<EOL>newResults = json.dumps(jobResults)<EOL>isUpdated = self._jobsDAO.jobSetFieldIfEqual(self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=jobResultsStr,<EOL>newValue=newResults)<EOL>if isUpdated or (not isUpdated and newResults==jobResultsStr):<EOL><INDENT>self._isBestModel = True<EOL>break<EOL><DEDENT><DEDENT>

Periodic check to see if this is the best model. This should only have an
effect if this is the *first* model to report its progress


def handleWarningSignal(self, signum, frame):

Handles a "warning signal" from the scheduler. This is received when the
scheduler is about to kill the the current process so that the worker can be
allocated to another job.

Right now, this function just sets the current model to the "Orphaned" state
in the models table so that another worker can eventually re-run this model

Parameters:
-----------------------------------------------------------------------


return self.__metricMgr.getMetricLabels()<EOL>

Returns:  A list of labels that correspond to metrics being computed


Delete the stored checkpoint for the specified modelID. This function is
called if the current model is now the best model, making the old model's
checkpoint obsolete

Parameters:
-----------------------------------------------------------------------
modelID:      The modelID for the checkpoint to delete. This is NOT the
              unique checkpointID


matchingKeys = matchPatterns([self._optimizeKeyPattern],<EOL>self._getMetricLabels())<EOL>if len(matchingKeys) == <NUM_LIT:0>:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>" %(self._optimizeKeyPattern, self._getMetricLabels()))<EOL><DEDENT>elif len(matchingKeys) > <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>" % (self._optimizeKeyPattern, matchingKeys))<EOL><DEDENT>return matchingKeys[<NUM_LIT:0>]<EOL>

Get the label for the metric being optimized. This function also caches
        the label in the instance variable self._optimizedMetricLabel

        Parameters:
        -----------------------------------------------------------------------
        metricLabels:   A sequence of all the labels being computed for this model

        Returns:        The label for the metric being optmized over


if not self.__predictionCache:<EOL><INDENT>return<EOL><DEDENT>if self._predictionLogger is None:<EOL><INDENT>self._createPredictionLogger()<EOL><DEDENT>startTime = time.time()<EOL>self._predictionLogger.writeRecords(self.__predictionCache,<EOL>progressCB=self.__writeRecordsCallback)<EOL>self._logger.info("<STR_LIT>",<EOL>len(self.__predictionCache), time.time() - startTime)<EOL>self.__predictionCache.clear()<EOL>

Writes the contents of this model's in-memory prediction cache to a permanent
store via the prediction output stream instance


<EOL>self._predictionLogger = BasicPredictionLogger(<EOL>fields=self._model.getFieldInfo(),<EOL>experimentDir=self._experimentDir,<EOL>label = "<STR_LIT>",<EOL>inferenceType=self._model.getInferenceType())<EOL>if self.__loggedMetricPatterns:<EOL><INDENT>metricLabels = self.__metricMgr.getMetricLabels()<EOL>loggedMetrics = matchPatterns(self.__loggedMetricPatterns, metricLabels)<EOL>self._predictionLogger.setLoggedMetrics(loggedMetrics)<EOL><DEDENT>


<EOL>print("<STR_LIT>", file=sys.stderr)<EOL>jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if jobCancel:<EOL><INDENT>self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED<EOL>self._isCanceled = True<EOL>self._logger.info("<STR_LIT>",<EOL>self._modelID, self._jobID)<EOL><DEDENT>else:<EOL><INDENT>stopReason = self._jobsDAO.modelsGetFields(self._modelID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if stopReason is None:<EOL><INDENT>pass<EOL><DEDENT>elif stopReason == ClientJobsDAO.STOP_REASON_KILLED:<EOL><INDENT>self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED<EOL>self._isKilled = True<EOL>self._logger.info("<STR_LIT>",<EOL>self._modelID)<EOL><DEDENT>elif stopReason == ClientJobsDAO.STOP_REASON_STOPPED:<EOL><INDENT>self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED<EOL>self._isCanceled = True<EOL>self._logger.info("<STR_LIT>", self._modelID)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError ("<STR_LIT>" % (stopReason))<EOL><DEDENT><DEDENT>

Check if the cancelation flag has been set for this model
        in the Model DB


<EOL>descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(<EOL>self._experimentDir)<EOL>expIface = helpers.getExperimentDescriptionInterfaceFromModule(<EOL>descriptionPyModule)<EOL>expIface.normalizeStreamSources()<EOL>modelDescription = expIface.getModelDescription()<EOL>self._modelControl = expIface.getModelControl()<EOL>streamDef = self._modelControl['<STR_LIT>']<EOL>from nupic.data.stream_reader import StreamReader<EOL>readTimeout = <NUM_LIT:0><EOL>self._inputSource = StreamReader(streamDef, isBlocking=False,<EOL>maxTimeout=readTimeout)<EOL>fieldStats = self._getFieldStats()<EOL>self._model = ModelFactory.create(modelDescription)<EOL>self._model.setFieldStatistics(fieldStats)<EOL>self._model.enableLearning()<EOL>self._model.enableInference(self._modelControl.get("<STR_LIT>", None))<EOL>self.__metricMgr = MetricsManager(self._modelControl.get('<STR_LIT>',None),<EOL>self._model.getFieldInfo(),<EOL>self._model.getInferenceType())<EOL>self.__loggedMetricPatterns = self._modelControl.get("<STR_LIT>", [])<EOL>self._optimizedMetricLabel = self.__getOptimizedMetricLabel()<EOL>self._reportMetricLabels = matchPatterns(self._reportKeyPatterns,<EOL>self._getMetricLabels())<EOL>self._periodic = self._initPeriodicActivities()<EOL>numIters = self._modelControl.get('<STR_LIT>', -<NUM_LIT:1>)<EOL>learningOffAt = None<EOL>iterationCountInferOnly = self._modelControl.get('<STR_LIT>', <NUM_LIT:0>)<EOL>if iterationCountInferOnly == -<NUM_LIT:1>:<EOL><INDENT>self._model.disableLearning()<EOL><DEDENT>elif iterationCountInferOnly > <NUM_LIT:0>:<EOL><INDENT>assert numIters > iterationCountInferOnly, "<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>learningOffAt = numIters - iterationCountInferOnly<EOL><DEDENT>self.__runTaskMainLoop(numIters, learningOffAt=learningOffAt)<EOL>self._finalize()<EOL>return (self._cmpReason, None)<EOL>

Runs the OPF Model

        Parameters:
        -------------------------------------------------------------------------
        retval:  (completionReason, completionMsg)
                  where completionReason is one of the ClientJobsDAO.CMPL_REASON_XXX
                    equates.


isSaved = False<EOL>while True:<EOL><INDENT>self._isBestModel, jobResults, jobResultsStr =self.__checkIfBestCompletedModel()<EOL>if self._isBestModel:<EOL><INDENT>if not isSaved:<EOL><INDENT>self.__flushPredictionCache()<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>self.__createModelCheckpoint()<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>isSaved = True<EOL><DEDENT>prevBest = jobResults.get('<STR_LIT>', None)<EOL>prevWasSaved = jobResults.get('<STR_LIT>', False)<EOL>if prevBest == self._modelID:<EOL><INDENT>assert not prevWasSaved<EOL><DEDENT>metrics = self._getMetrics()<EOL>jobResults['<STR_LIT>'] = self._modelID<EOL>jobResults['<STR_LIT>'] = metrics[self._optimizedMetricLabel]<EOL>jobResults['<STR_LIT>'] = metrics<EOL>jobResults['<STR_LIT>'] = True<EOL>isUpdated = self._jobsDAO.jobSetFieldIfEqual(self._jobID,<EOL>fieldName='<STR_LIT>',<EOL>curValue=jobResultsStr,<EOL>newValue=json.dumps(jobResults))<EOL>if isUpdated:<EOL><INDENT>if prevWasSaved:<EOL><INDENT>self.__deleteOutputCache(prevBest)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>self.__deleteModelCheckpoint(prevBest)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL><DEDENT>self._logger.info("<STR_LIT>", self._modelID)<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.__deleteOutputCache(self._modelID)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>self.__deleteModelCheckpoint(self._modelID)<EOL>self._jobsDAO.modelUpdateTimestamp(self._modelID)<EOL>break<EOL><DEDENT><DEDENT>

Check if this is the best model
If so:
  1) Write it's checkpoint
  2) Record this model as the best
  3) Delete the previous best's output cache
Otherwise:
  1) Delete our output cache


maxDelay = <NUM_LIT:0><EOL>for inferenceElement, inference in inferences.iteritems():<EOL><INDENT>if isinstance(inference, dict):<EOL><INDENT>for key in inference.iterkeys():<EOL><INDENT>maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement,<EOL>key),<EOL>maxDelay)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement),<EOL>maxDelay)<EOL><DEDENT><DEDENT>return maxDelay<EOL>

Returns the maximum delay for the InferenceElements in the inference
dictionary

Parameters:
-----------------------------------------------------------------------
inferences:   A dictionary where the keys are InferenceElements