def runModelGivenBaseAndParams(modelID, jobID, baseDescription, params,<EOL>predictedField, reportKeys, optimizeKey, jobsDAO,<EOL>modelCheckpointGUID, logLevel=None, predictionCacheMaxRecords=None):

from nupic.swarming.ModelRunner import OPFModelRunner<EOL>logger = logging.getLogger('<STR_LIT>')<EOL>experimentDir = tempfile.mkdtemp()<EOL>try:<EOL><INDENT>logger.info("<STR_LIT>" % (experimentDir))<EOL>paramsFilePath = os.path.join(experimentDir, '<STR_LIT>')<EOL>paramsFile = open(paramsFilePath, '<STR_LIT:wb>')<EOL>paramsFile.write(_paramsFileHead())<EOL>items = list(params.items())<EOL>items.sort()<EOL>for (key,value) in items:<EOL><INDENT>quotedKey = _quoteAndEscape(key)<EOL>if isinstance(value, str):<EOL><INDENT>paramsFile.write("<STR_LIT>" % (quotedKey , value))<EOL><DEDENT>else:<EOL><INDENT>paramsFile.write("<STR_LIT>" % (quotedKey , value))<EOL><DEDENT><DEDENT>paramsFile.write(_paramsFileTail())<EOL>paramsFile.close()<EOL>baseParamsFile = open(os.path.join(experimentDir, '<STR_LIT>'), '<STR_LIT:wb>')<EOL>baseParamsFile.write(baseDescription)<EOL>baseParamsFile.close()<EOL>fd = open(paramsFilePath)<EOL>expDescription = fd.read()<EOL>fd.close()<EOL>jobsDAO.modelSetFields(modelID, {'<STR_LIT>': expDescription})<EOL>try:<EOL><INDENT>runner = OPFModelRunner(<EOL>modelID=modelID,<EOL>jobID=jobID,<EOL>predictedField=predictedField,<EOL>experimentDir=experimentDir,<EOL>reportKeyPatterns=reportKeys,<EOL>optimizeKeyPattern=optimizeKey,<EOL>jobsDAO=jobsDAO,<EOL>modelCheckpointGUID=modelCheckpointGUID,<EOL>logLevel=logLevel,<EOL>predictionCacheMaxRecords=predictionCacheMaxRecords)<EOL>signal.signal(signal.SIGINT, runner.handleWarningSignal)<EOL>(completionReason, completionMsg) = runner.run()<EOL><DEDENT>except InvalidConnectionException:<EOL><INDENT>raise<EOL><DEDENT>except Exception as e:<EOL><INDENT>(completionReason, completionMsg) = _handleModelRunnerException(jobID,<EOL>modelID, jobsDAO, experimentDir, logger, e)<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>shutil.rmtree(experimentDir)<EOL>signal.signal(signal.SIGINT, signal.default_int_handler)<EOL><DEDENT>return (completionReason, completionMsg)<EOL>

This creates an experiment directory with a base.py description file
    created from 'baseDescription' and a description.py generated from the
    given params dict and then runs the experiment.

    Parameters:
    -------------------------------------------------------------------------
    modelID:              ID for this model in the models table
    jobID:                ID for this hypersearch job in the jobs table
    baseDescription:      Contents of a description.py with the base experiment
                                            description
    params:               Dictionary of specific parameters to override within
                                    the baseDescriptionFile.
    predictedField:       Name of the input field for which this model is being
                                      optimized
    reportKeys:           Which metrics of the experiment to store into the
                                      results dict of the model's database entry
    optimizeKey:          Which metric we are optimizing for
    jobsDAO               Jobs data access object - the interface to the
                                    jobs database which has the model's table.
    modelCheckpointGUID:  A persistent, globally-unique identifier for
                                    constructing the model checkpoint key
    logLevel:             override logging level to this value, if not None

    retval:               (completionReason, completionMsg)


return "<STR_LIT>" + str(uuid.uuid1())<EOL>


def rCopy(d, f=identityConversion, discardNoneKeys=True, deepCopy=True):

<EOL>if deepCopy:<EOL><INDENT>d = copy.deepcopy(d)<EOL><DEDENT>newDict = {}<EOL>toCopy = [(k, v, newDict, ()) for k, v in d.items()]<EOL>while len(toCopy) > <NUM_LIT:0>:<EOL><INDENT>k, v, d, prevKeys = toCopy.pop()<EOL>prevKeys = prevKeys + (k,)<EOL>if isinstance(v, dict):<EOL><INDENT>d[k] = dict()<EOL>toCopy[<NUM_LIT:0>:<NUM_LIT:0>] = [(innerK, innerV, d[k], prevKeys)<EOL>for innerK, innerV in v.items()]<EOL><DEDENT>else:<EOL><INDENT>newV = f(v, prevKeys)<EOL>if not discardNoneKeys or newV is not None:<EOL><INDENT>d[k] = newV<EOL><DEDENT><DEDENT><DEDENT>return newDict<EOL>

Recursively copies a dict and returns the result.

    Args:
      d: The dict to copy.
      f: A function to apply to values when copying that takes the value and the
          list of keys from the root of the dict to the value and returns a value
          for the new dict.
      discardNoneKeys: If True, discard key-value pairs when f returns None for
          the value.
      deepCopy: If True, all values in returned dict are true copies (not the
          same object).
    Returns:
      A new dict with keys and values from d replaced with the result of f.


remainingDicts = [(d, ())]<EOL>while len(remainingDicts) > <NUM_LIT:0>:<EOL><INDENT>current, prevKeys = remainingDicts.pop()<EOL>for k, v in current.items():<EOL><INDENT>keys = prevKeys + (k,)<EOL>if isinstance(v, dict):<EOL><INDENT>remainingDicts.insert(<NUM_LIT:0>, (v, keys))<EOL><DEDENT>else:<EOL><INDENT>f(v, keys)<EOL><DEDENT><DEDENT><DEDENT>

Recursively applies f to the values in dict d.

    Args:
      d: The dict to recurse over.
      f: A function to apply to values in d that takes the value and a list of
          keys from the root of the dict to the value.


<EOL>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj = obj._asdict()<EOL><DEDENT>if isinstance(obj, dict):<EOL><INDENT>objOut = dict()<EOL>for key,val in obj.items():<EOL><INDENT>objOut[key] = clippedObj(val)<EOL><DEDENT><DEDENT>elif hasattr(obj, '<STR_LIT>'):<EOL><INDENT>objOut = []<EOL>for val in obj:<EOL><INDENT>objOut.append(clippedObj(val))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>objOut = str(obj)<EOL>if len(objOut) > maxElementSize:<EOL><INDENT>objOut = objOut[<NUM_LIT:0>:maxElementSize] + '<STR_LIT>'<EOL><DEDENT><DEDENT>return objOut<EOL>

Return a clipped version of obj suitable for printing, This
is useful when generating log messages by printing data structures, but
don't want the message to be too long.

If passed in a dict, list, or namedtuple, each element of the structure's
string representation will be limited to 'maxElementSize' characters. This
will return a new object where the string representation of each element
has been truncated to fit within maxElementSize.


assert len(list(kwds.keys())) >= <NUM_LIT:1><EOL>assert '<STR_LIT>' in kwds or '<STR_LIT>' in kwds<EOL>schemaDict = None<EOL>if '<STR_LIT>' in kwds:<EOL><INDENT>schemaPath = kwds.pop('<STR_LIT>')<EOL>schemaDict = loadJsonValueFromFile(schemaPath)<EOL><DEDENT>elif '<STR_LIT>' in kwds:<EOL><INDENT>schemaDict = kwds.pop('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>validictory.validate(value, schemaDict, **kwds)<EOL><DEDENT>except validictory.ValidationError as e:<EOL><INDENT>raise ValidationError(e)<EOL><DEDENT>

Validate a python value against json schema:
    validate(value, schemaPath)
    validate(value, schemaDict)

    value:          python object to validate against the schema

    The json schema may be specified either as a path of the file containing
    the json schema or as a python dictionary using one of the
    following keywords as arguments:
      schemaPath:     Path of file containing the json schema object.
      schemaDict:     Python dictionary containing the json schema object

    Returns: nothing

    Raises:
            ValidationError when value fails json validation


with open(inputFilePath) as fileObj:<EOL><INDENT>value = json.load(fileObj)<EOL><DEDENT>return value<EOL>

Loads a json value from a file and converts it to the corresponding python
    object.

    inputFilePath:
                    Path of the json file;

    Returns:
                    python value that represents the loaded json value


itemStrs = []<EOL>if isinstance(obj, dict):<EOL><INDENT>items = list(obj.items())<EOL>items.sort()<EOL>for key, value in items:<EOL><INDENT>itemStrs.append('<STR_LIT>' % (json.dumps(key), sortedJSONDumpS(value)))<EOL><DEDENT>return '<STR_LIT>' % ('<STR_LIT:U+002CU+0020>'.join(itemStrs))<EOL><DEDENT>elif hasattr(obj, '<STR_LIT>'):<EOL><INDENT>for val in obj:<EOL><INDENT>itemStrs.append(sortedJSONDumpS(val))<EOL><DEDENT>return '<STR_LIT>' % ('<STR_LIT:U+002CU+0020>'.join(itemStrs))<EOL><DEDENT>else:<EOL><INDENT>return json.dumps(obj)<EOL><DEDENT>

Return a JSON representation of obj with sorted keys on any embedded dicts.
This insures that the same object will always be represented by the same
string even if it contains dicts (where the sort order of the keys is
normally undefined).


self.__activities = []<EOL>for req in requestedActivities:<EOL><INDENT>act =   self.Activity(repeating=req.repeating,<EOL>period=req.period,<EOL>cb=req.cb,<EOL>iteratorHolder=[iter(range(req.period))])<EOL>self.__activities.append(act)<EOL><DEDENT>return<EOL>

requestedActivities: a sequence of PeriodicActivityRequest elements


<EOL>for act in self.__activities:<EOL><INDENT>if not act.iteratorHolder[<NUM_LIT:0>]:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>next(act.iteratorHolder[<NUM_LIT:0>])<EOL><DEDENT>except StopIteration:<EOL><INDENT>act.cb()<EOL>if act.repeating:<EOL><INDENT>act.iteratorHolder[<NUM_LIT:0>] = iter(range(act.period))<EOL><DEDENT>else:<EOL><INDENT>act.iteratorHolder[<NUM_LIT:0>] = None<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Activity tick handler; services all activities

        Returns:      True if controlling iterator says it's okay to keep going;
                      False to stop


assert isinstance(s, str),"<STR_LIT>" % (type(str), type(s), s)<EOL>s = s.replace("<STR_LIT:\\>", "<STR_LIT>")<EOL>s = s.replace("<STR_LIT:\n>", "<STR_LIT>")<EOL>s = s.replace("<STR_LIT:\t>", "<STR_LIT>")<EOL>s = s.replace("<STR_LIT:U+002C>", "<STR_LIT:\t>")<EOL>return s<EOL>

Escape commas, tabs, newlines and dashes in a string

    Commas are encoded as tabs


process = subprocess.Popen(["<STR_LIT>", "<STR_LIT>"], stdout=subprocess.PIPE)<EOL>stdout = process.communicate()[<NUM_LIT:0>]<EOL>result = process.returncode<EOL>if result != <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>running = False<EOL>for line in stdout.split("<STR_LIT:\n>"):<EOL><INDENT>if "<STR_LIT>" in line and "<STR_LIT>" in line:<EOL><INDENT>running = True<EOL>break<EOL><DEDENT><DEDENT>return running<EOL>

Return true if the engine services are running


def runWithConfig(swarmConfig, options,<EOL>outDir=None, outputLabel="<STR_LIT:default>",<EOL>permWorkDir=None, verbosity=<NUM_LIT:1>):

global g_currentVerbosityLevel<EOL>g_currentVerbosityLevel = verbosity<EOL>if outDir is None:<EOL><INDENT>outDir = os.getcwd()<EOL><DEDENT>if permWorkDir is None:<EOL><INDENT>permWorkDir = os.getcwd()<EOL><DEDENT>_checkOverwrite(options, outDir)<EOL>_generateExpFilesFromSwarmDescription(swarmConfig, outDir)<EOL>options["<STR_LIT>"] = swarmConfig<EOL>options["<STR_LIT>"] = outputLabel<EOL>options["<STR_LIT>"] = outDir<EOL>options["<STR_LIT>"] = permWorkDir<EOL>runOptions = _injectDefaultOptions(options)<EOL>_validateOptions(runOptions)<EOL>return _runAction(runOptions)<EOL>

Starts a swarm, given an dictionary configuration.
@param swarmConfig {dict} A complete [swarm description](http://nupic.docs.numenta.org/0.7.0.dev0/guides/swarming/running.html#the-swarm-description) object.
@param outDir {string} Optional path to write swarm details (defaults to
                       current working directory).
@param outputLabel {string} Optional label for output (defaults to "default").
@param permWorkDir {string} Optional location of working directory (defaults
                            to current working directory).
@param verbosity {int} Optional (1,2,3) increasing verbosity of output.

@returns {object} Model parameters


def runWithJsonFile(expJsonFilePath, options, outputLabel, permWorkDir):

if "<STR_LIT>" in options:<EOL><INDENT>verbosity = options["<STR_LIT>"]<EOL>del options["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>verbosity = <NUM_LIT:1><EOL><DEDENT>_setupInterruptHandling()<EOL>with open(expJsonFilePath, "<STR_LIT:r>") as jsonFile:<EOL><INDENT>expJsonConfig = json.loads(jsonFile.read())<EOL><DEDENT>outDir = os.path.dirname(expJsonFilePath)<EOL>return runWithConfig(expJsonConfig, options, outDir=outDir,<EOL>outputLabel=outputLabel, permWorkDir=permWorkDir,<EOL>verbosity=verbosity)<EOL>

Starts a swarm, given a path to a JSON file containing configuration.

This function is meant to be used with a CLI wrapper that passes command line
arguments in through the options parameter.

@param expJsonFilePath {string} Path to a JSON file containing the complete
                               [swarm description](http://nupic.docs.numenta.org/0.7.0.dev0/guides/swarming/running.html#the-swarm-description).
@param options {dict} CLI options.
@param outputLabel {string} Label for output.
@param permWorkDir {string} Location of working directory.

@returns {int} Swarm job id.


def runWithPermutationsScript(permutationsFilePath, options,<EOL>outputLabel, permWorkDir):

global g_currentVerbosityLevel<EOL>if "<STR_LIT>" in options:<EOL><INDENT>g_currentVerbosityLevel = options["<STR_LIT>"]<EOL>del options["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>g_currentVerbosityLevel = <NUM_LIT:1><EOL><DEDENT>_setupInterruptHandling()<EOL>options["<STR_LIT>"] = permutationsFilePath<EOL>options["<STR_LIT>"] = outputLabel<EOL>options["<STR_LIT>"] = permWorkDir<EOL>options["<STR_LIT>"] = permWorkDir<EOL>runOptions = _injectDefaultOptions(options)<EOL>_validateOptions(runOptions)<EOL>return _runAction(runOptions)<EOL>

Starts a swarm, given a path to a permutations.py script.

This function is meant to be used with a CLI wrapper that passes command line
arguments in through the options parameter.

@param permutationsFilePath {string} Path to permutations.py.
@param options {dict} CLI options.
@param outputLabel {string} Label for output.
@param permWorkDir {string} Location of working directory.

@returns {object} Model parameters.


raise DeprecationWarning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>


return cjdao.ClientJobsDAO.get()<EOL>

Returns: The shared cjdao.ClientJobsDAO instance


<EOL>return False<EOL>

Check whether any experiments failed in our latest hypersearch

    Parameters:
      hyperSearchJob: _HyperSearchJob instance; if None, will get it from saved
                      jobID, if any

    Returns: False if all models succeeded, True if one or more had errors


assert os.path.exists(filePath)<EOL>stampNum = <NUM_LIT:0><EOL>(prefix, suffix) = os.path.splitext(filePath)<EOL>while True:<EOL><INDENT>backupPath = "<STR_LIT>" % (prefix, stampNum, suffix)<EOL>stampNum += <NUM_LIT:1><EOL>if not os.path.exists(backupPath):<EOL><INDENT>break<EOL><DEDENT><DEDENT>shutil.copyfile(filePath, backupPath)<EOL>return backupPath<EOL>


return next(_iterModels([nupicModelID]))<EOL>

A convenience function that retrieves inforamtion about a single model

    See also: _iterModels()

    Parameters:
    ----------------------------------------------------------------------
    nupicModelID:      Nupic modelID
    retval:           _NupicModelInfo instance for the given nupicModelID.


class ModelInfoIterator(object):<EOL><INDENT>"""<STR_LIT>"""<EOL>__CACHE_LIMIT = <NUM_LIT:1000><EOL>debug=False<EOL>def __init__(self, modelIDs):<EOL><INDENT>"""<STR_LIT>"""<EOL>self.__modelIDs = tuple(modelIDs)<EOL>if self.debug:<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % len(self.__modelIDs))<EOL><DEDENT>self.__nextIndex = <NUM_LIT:0><EOL>self.__modelCache = collections.deque()<EOL>return<EOL><DEDENT>def __iter__(self):<EOL><INDENT>"""<STR_LIT>"""<EOL>return self<EOL><DEDENT>def __next__(self):<EOL><INDENT>"""<STR_LIT>"""<EOL>return self.__getNext()<EOL><DEDENT>def __getNext(self):<EOL><INDENT>"""<STR_LIT>"""<EOL>if self.debug:<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (<EOL>len(self.__modelCache)))<EOL><DEDENT>if not self.__modelCache:<EOL><INDENT>self.__fillCache()<EOL><DEDENT>if not self.__modelCache:<EOL><INDENT>raise StopIteration()<EOL><DEDENT>return self.__modelCache.popleft()<EOL><DEDENT>def __fillCache(self):<EOL><INDENT>"""<STR_LIT>"""<EOL>assert (not self.__modelCache)<EOL>numModelIDs = len(self.__modelIDs) if self.__modelIDs else <NUM_LIT:0><EOL>if self.__nextIndex >= numModelIDs:<EOL><INDENT>return<EOL><DEDENT>idRange = self.__nextIndex + self.__CACHE_LIMIT<EOL>if idRange > numModelIDs:<EOL><INDENT>idRange = numModelIDs<EOL><DEDENT>lookupIDs = self.__modelIDs[self.__nextIndex:idRange]<EOL>self.__nextIndex += (idRange - self.__nextIndex)<EOL>infoList = _clientJobsDB().modelsInfo(lookupIDs)<EOL>assert len(infoList) == len(lookupIDs),"<STR_LIT>" %(len(infoList), len(lookupIDs))<EOL>for rawInfo in infoList:<EOL><INDENT>modelInfo = _NupicModelInfo(rawInfo=rawInfo)<EOL>self.__modelCache.append(modelInfo)<EOL><DEDENT>assert len(self.__modelCache) == len(lookupIDs),"<STR_LIT>" %(len(self.__modelCache), len(lookupIDs))<EOL>if self.debug:<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" %(len(self.__modelCache),))<EOL><DEDENT><DEDENT><DEDENT>return ModelInfoIterator(modelIDs)<EOL>

Creates an iterator that returns ModelInfo elements for the given modelIDs

    WARNING:      The order of ModelInfo elements returned by the iterator
                  may not match the order of the given modelIDs

    Parameters:
    ----------------------------------------------------------------------
    modelIDs:       A sequence of model identifiers (e.g., as returned by
                    _HyperSearchJob.queryModelIDs()).
    retval:         Iterator that returns ModelInfo elements for the given
                    modelIDs (NOTE:possibly in a different order)


self.__cjDAO = _clientJobsDB()<EOL>self._options = options<EOL>self.__searchJob = None<EOL>self.__foundMetrcsKeySet = set()<EOL>self._workers = None<EOL>return<EOL>


self.__searchJob = self.__startSearch()<EOL>self.monitorSearchJob()<EOL>


self.__searchJob = self.loadSavedHyperSearchJob(<EOL>permWorkDir=self._options["<STR_LIT>"],<EOL>outputLabel=self._options["<STR_LIT>"])<EOL>self.monitorSearchJob()<EOL>

Pick up the latest search from a saved jobID and monitor it to completion
        Parameters:
        ----------------------------------------------------------------------
        retval:         nothing


assert self.__searchJob is not None<EOL>jobID = self.__searchJob.getJobID()<EOL>startTime = time.time()<EOL>lastUpdateTime = datetime.now()<EOL>expectedNumModels = self.__searchJob.getExpectedNumModels(<EOL>searchMethod = self._options["<STR_LIT>"])<EOL>lastNumFinished = <NUM_LIT:0><EOL>finishedModelIDs = set()<EOL>finishedModelStats = _ModelStats()<EOL>lastWorkerState = None<EOL>lastJobResults = None<EOL>lastModelMilestones = None<EOL>lastEngStatus = None<EOL>hyperSearchFinished = False<EOL>while not hyperSearchFinished:<EOL><INDENT>jobInfo = self.__searchJob.getJobStatus(self._workers)<EOL>hyperSearchFinished = jobInfo.isFinished()<EOL>modelIDs = self.__searchJob.queryModelIDs()<EOL>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (<EOL>len(modelIDs), len(finishedModelIDs)))<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>checkModelIDs = []<EOL>for modelID in modelIDs:<EOL><INDENT>if modelID not in finishedModelIDs:<EOL><INDENT>checkModelIDs.append(modelID)<EOL><DEDENT><DEDENT>del modelIDs<EOL>if checkModelIDs:<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (len(checkModelIDs)))<EOL>errorCompletionMsg = None<EOL>for (i, modelInfo) in enumerate(_iterModels(checkModelIDs)):<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (i, modelInfo))<EOL>if modelInfo.isFinished():<EOL><INDENT>finishedModelIDs.add(modelInfo.getModelID())<EOL>finishedModelStats.update(modelInfo)<EOL>if (modelInfo.getCompletionReason().isError() and<EOL>not errorCompletionMsg):<EOL><INDENT>errorCompletionMsg = modelInfo.getCompletionMsg()<EOL><DEDENT>metrics = modelInfo.getReportMetrics()<EOL>self.__foundMetrcsKeySet.update(list(metrics.keys()))<EOL><DEDENT><DEDENT><DEDENT>numFinished = len(finishedModelIDs)<EOL>if numFinished != lastNumFinished:<EOL><INDENT>lastNumFinished = numFinished<EOL>if expectedNumModels is None:<EOL><INDENT>expModelsStr = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>expModelsStr = "<STR_LIT>" % (expectedNumModels)<EOL><DEDENT>stats = finishedModelStats<EOL>print(("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>jobID,<EOL>numFinished,<EOL>expModelsStr,<EOL>(stats.numCompletedEOF+stats.numCompletedStopped),<EOL>"<STR_LIT>" if stats.numCompletedEOF else "<STR_LIT>",<EOL>stats.numCompletedEOF,<EOL>"<STR_LIT>" if stats.numCompletedStopped else "<STR_LIT>",<EOL>stats.numCompletedStopped,<EOL>"<STR_LIT>" if stats.numCompletedKilled else "<STR_LIT>",<EOL>stats.numCompletedKilled,<EOL>"<STR_LIT>" if stats.numCompletedError else "<STR_LIT:error>",<EOL>stats.numCompletedError,<EOL>"<STR_LIT>" if stats.numCompletedError else "<STR_LIT>",<EOL>stats.numCompletedOrphaned,<EOL>"<STR_LIT>" if stats.numCompletedOther else "<STR_LIT>",<EOL>stats.numCompletedOther)))<EOL>if errorCompletionMsg:<EOL><INDENT>print("<STR_LIT>" % errorCompletionMsg)<EOL><DEDENT><DEDENT>workerState = jobInfo.getWorkerState()<EOL>if workerState != lastWorkerState:<EOL><INDENT>print("<STR_LIT>" % (pprint.pformat(workerState,<EOL>indent=<NUM_LIT:4>)))<EOL>lastWorkerState = workerState<EOL><DEDENT>jobResults = jobInfo.getResults()<EOL>if jobResults != lastJobResults:<EOL><INDENT>print("<STR_LIT>"% (pprint.pformat(jobResults, indent=<NUM_LIT:4>), time.time()-startTime))<EOL>lastJobResults = jobResults<EOL><DEDENT>modelMilestones = jobInfo.getModelMilestones()<EOL>if modelMilestones != lastModelMilestones:<EOL><INDENT>print("<STR_LIT>" % (<EOL>pprint.pformat(modelMilestones, indent=<NUM_LIT:4>)))<EOL>lastModelMilestones = modelMilestones<EOL><DEDENT>engStatus = jobInfo.getEngStatus()<EOL>if engStatus != lastEngStatus:<EOL><INDENT>print("<STR_LIT>" % (engStatus))<EOL>lastEngStatus = engStatus<EOL><DEDENT><DEDENT>if not hyperSearchFinished:<EOL><INDENT>if self._options["<STR_LIT>"] != None:<EOL><INDENT>if ((datetime.now() - lastUpdateTime) ><EOL>timedelta(minutes=self._options["<STR_LIT>"])):<EOL><INDENT>print("<STR_LIT>")<EOL>self.__cjDAO.jobCancel(jobID)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>time.sleep(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>modelIDs = self.__searchJob.queryModelIDs()<EOL>print("<STR_LIT>" % len(modelIDs))<EOL>print("<STR_LIT>")<EOL>jobInfo = self.__searchJob.getJobStatus(self._workers)<EOL>print("<STR_LIT>" % (jobInfo.getWorkerCompletionMsg()))<EOL>


def _launchWorkers(self, cmdLine, numWorkers):

self._workers = []<EOL>for i in range(numWorkers):<EOL><INDENT>stdout = tempfile.NamedTemporaryFile(delete=False)<EOL>stderr = tempfile.NamedTemporaryFile(delete=False)<EOL>p = subprocess.Popen(cmdLine, bufsize=<NUM_LIT:1>, env=os.environ, shell=True,<EOL>stdin=None, stdout=stdout, stderr=stderr)<EOL>p._stderr_file = stderr<EOL>p._stdout_file = stdout<EOL>self._workers.append(p)<EOL><DEDENT>

Launch worker processes to execute the given command line

        Parameters:
        -----------------------------------------------
        cmdLine: The command line for each worker
        numWorkers: number of workers to launch


<EOL>params = _ClientJobUtils.makeSearchJobParamsDict(options=self._options,<EOL>forRunning=True)<EOL>if self._options["<STR_LIT:action>"] == "<STR_LIT>":<EOL><INDENT>args = [sys.argv[<NUM_LIT:0>], "<STR_LIT>" % (json.dumps(params))]<EOL>print()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>jobID = hypersearch_worker.main(args)<EOL><DEDENT>else:<EOL><INDENT>cmdLine = _setUpExports(self._options["<STR_LIT>"])<EOL>cmdLine += "<STR_LIT>"<EOL>maxWorkers = self._options["<STR_LIT>"]<EOL>jobID = self.__cjDAO.jobInsert(<EOL>client="<STR_LIT>",<EOL>cmdLine=cmdLine,<EOL>params=json.dumps(params),<EOL>minimumWorkers=<NUM_LIT:1>,<EOL>maximumWorkers=maxWorkers,<EOL>jobType=self.__cjDAO.JOB_TYPE_HS)<EOL>cmdLine = "<STR_LIT>""<STR_LIT>" % (jobID)<EOL>self._launchWorkers(cmdLine, maxWorkers)<EOL><DEDENT>searchJob = _HyperSearchJob(jobID)<EOL>self.__saveHyperSearchJobID(<EOL>permWorkDir=self._options["<STR_LIT>"],<EOL>outputLabel=self._options["<STR_LIT>"],<EOL>hyperSearchJob=searchJob)<EOL>if self._options["<STR_LIT:action>"] == "<STR_LIT>":<EOL><INDENT>print("<STR_LIT>" % (jobID))<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>" % (jobID))<EOL>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (cmdLine,))<EOL><DEDENT>return searchJob<EOL>

Starts HyperSearch as a worker or runs it inline for the "dryRun" action

        Parameters:
        ----------------------------------------------------------------------
        retval:         the new _HyperSearchJob instance representing the
                        HyperSearch job


assert self.__searchJob is not None<EOL>return self.__searchJob<EOL>

Retrieves the runner's _HyperSearchJob instance; NOTE: only available
        after run().

        Parameters:
        ----------------------------------------------------------------------
        retval:         _HyperSearchJob instance or None


return tuple(self.__foundMetrcsKeySet)<EOL>

Returns a tuple of all metrics keys discovered while running HyperSearch.

        NOTE: This is an optimization so that our client may
            use this info for generating the report csv file without having
            to pre-scan all modelInfos

        Parameters:
        ----------------------------------------------------------------------
        retval:         Tuple of metrics keys discovered while running
                        HyperSearch;


print("<STR_LIT>")<EOL>searchParams = _ClientJobUtils.makeSearchJobParamsDict(options=options)<EOL>

Prints a listing of experiments that would take place without
        actually executing them.

        Parameters:
        ----------------------------------------------------------------------
        options:        NupicRunPermutations options dict
        retval:         nothing


<EOL>if hyperSearchJob is None:<EOL><INDENT>hyperSearchJob = cls.loadSavedHyperSearchJob(<EOL>permWorkDir=options["<STR_LIT>"],<EOL>outputLabel=options["<STR_LIT>"])<EOL><DEDENT>modelIDs = hyperSearchJob.queryModelIDs()<EOL>bestModel = None<EOL>metricstmp = set()<EOL>searchVar = set()<EOL>for modelInfo in _iterModels(modelIDs):<EOL><INDENT>if modelInfo.isFinished():<EOL><INDENT>vars = list(modelInfo.getParamLabels().keys())<EOL>searchVar.update(vars)<EOL>metrics = modelInfo.getReportMetrics()<EOL>metricstmp.update(list(metrics.keys()))<EOL><DEDENT><DEDENT>if metricsKeys is None:<EOL><INDENT>metricsKeys = metricstmp<EOL><DEDENT>reportWriter = _ReportCSVWriter(hyperSearchJob=hyperSearchJob,<EOL>metricsKeys=metricsKeys,<EOL>searchVar=searchVar,<EOL>outputDirAbsPath=options["<STR_LIT>"],<EOL>outputLabel=options["<STR_LIT>"],<EOL>replaceReport=replaceReport)<EOL>modelStats = _ModelStats()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>searchParams = hyperSearchJob.getParams()<EOL>(optimizationMetricKey, maximizeMetric) = (<EOL>_PermutationUtils.getOptimizationMetricInfo(searchParams))<EOL>formatStr = None<EOL>foundMetricsKeySet = set(metricsKeys)<EOL>sortedMetricsKeys = []<EOL>jobInfo = _clientJobsDB().jobInfo(hyperSearchJob.getJobID())<EOL>if jobInfo.cancel == <NUM_LIT:1>:<EOL><INDENT>raise Exception(jobInfo.workerCompletionMsg)<EOL><DEDENT>try:<EOL><INDENT>results = json.loads(jobInfo.results)<EOL><DEDENT>except Exception as e:<EOL><INDENT>print("<STR_LIT>""<STR_LIT>")<EOL>print("<STR_LIT>", jobInfo)<EOL>print("<STR_LIT>", jobInfo.results)<EOL>print("<STR_LIT>", e)<EOL>raise<EOL><DEDENT>bestModelNum = results["<STR_LIT>"]<EOL>bestModelIterIndex = None<EOL>totalWallTime = <NUM_LIT:0><EOL>totalRecords = <NUM_LIT:0><EOL>scoreModelIDDescList = []<EOL>for (i, modelInfo) in enumerate(_iterModels(modelIDs)):<EOL><INDENT>reportWriter.emit(modelInfo)<EOL>totalRecords+=modelInfo.getNumRecords()<EOL>format = "<STR_LIT>"<EOL>startTime = modelInfo.getStartTime()<EOL>if modelInfo.isFinished():<EOL><INDENT>endTime = modelInfo.getEndTime()<EOL>st = datetime.strptime(startTime, format)<EOL>et = datetime.strptime(endTime, format)<EOL>totalWallTime+=(et-st).seconds<EOL><DEDENT>modelStats.update(modelInfo)<EOL>expDesc = modelInfo.getModelDescription()<EOL>reportMetrics = modelInfo.getReportMetrics()<EOL>optimizationMetrics = modelInfo.getOptimizationMetrics()<EOL>if modelInfo.getModelID() == bestModelNum:<EOL><INDENT>bestModel = modelInfo<EOL>bestModelIterIndex=i<EOL>bestMetric = list(optimizationMetrics.values())[<NUM_LIT:0>]<EOL><DEDENT>if optimizationMetrics:<EOL><INDENT>assert len(optimizationMetrics) == <NUM_LIT:1>, (<EOL>"<STR_LIT>" % (<EOL>len(optimizationMetrics), optimizationMetrics, modelInfo))<EOL><DEDENT>if modelInfo.getCompletionReason().isEOF():<EOL><INDENT>scoreModelIDDescList.append((list(optimizationMetrics.values())[<NUM_LIT:0>],<EOL>modelInfo.getModelID(),<EOL>modelInfo.getGeneratedDescriptionFile(),<EOL>modelInfo.getParamLabels()))<EOL><DEDENT>print("<STR_LIT>" % (i, modelInfo, expDesc))<EOL>if (modelInfo.isFinished() and<EOL>not (modelInfo.getCompletionReason().isStopped or<EOL>modelInfo.getCompletionReason().isEOF())):<EOL><INDENT>print("<STR_LIT>" % modelInfo.getCompletionMsg())<EOL><DEDENT>if reportMetrics:<EOL><INDENT>foundMetricsKeySet.update(iter(reportMetrics.keys()))<EOL>if len(sortedMetricsKeys) != len(foundMetricsKeySet):<EOL><INDENT>sortedMetricsKeys = sorted(foundMetricsKeySet)<EOL>maxKeyLen = max([len(k) for k in sortedMetricsKeys])<EOL>formatStr = "<STR_LIT>" % (maxKeyLen+<NUM_LIT:2>)<EOL><DEDENT>for key in sortedMetricsKeys:<EOL><INDENT>if key in reportMetrics:<EOL><INDENT>if key == optimizationMetricKey:<EOL><INDENT>m = "<STR_LIT>" % reportMetrics[key]<EOL><DEDENT>else:<EOL><INDENT>m = "<STR_LIT>" % reportMetrics[key]<EOL><DEDENT>print(formatStr % (key+"<STR_LIT::>"), m)<EOL><DEDENT><DEDENT>print()<EOL><DEDENT><DEDENT>print("<STR_LIT>")<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>len(modelIDs),<EOL>("<STR_LIT>"<EOL>if (modelStats.numCompletedKilled + modelStats.numCompletedEOF) ==<EOL>len(modelIDs)<EOL>else "<STR_LIT>" % (<EOL>len(modelIDs) - (<EOL>modelStats.numCompletedKilled + modelStats.numCompletedEOF +<EOL>modelStats.numCompletedStopped)))))<EOL>if modelStats.numStatusOther > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>modelStats.numStatusOther))<EOL><DEDENT>print("<STR_LIT>" % modelStats.numStatusWaitingToStart)<EOL>print("<STR_LIT>" % modelStats.numStatusRunning)<EOL>print("<STR_LIT>" % modelStats.numStatusCompleted)<EOL>if modelStats.numCompletedOther > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>modelStats.numCompletedOther))<EOL><DEDENT>print("<STR_LIT>" % modelStats.numCompletedEOF)<EOL>print("<STR_LIT>" % modelStats.numCompletedStopped)<EOL>print("<STR_LIT>" % modelStats.numCompletedOrphaned)<EOL>print("<STR_LIT>" % modelStats.numCompletedKilled)<EOL>print("<STR_LIT>" % modelStats.numCompletedError)<EOL>assert modelStats.numStatusOther == <NUM_LIT:0>, "<STR_LIT>" % (<EOL>modelStats.numStatusOther)<EOL>assert modelStats.numCompletedOther == <NUM_LIT:0>, "<STR_LIT>" % (<EOL>modelStats.numCompletedOther)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>print()<EOL>global gCurrentSearch<EOL>jobStatus = hyperSearchJob.getJobStatus(gCurrentSearch._workers)<EOL>jobResults = jobStatus.getResults()<EOL>if "<STR_LIT>" in jobResults:<EOL><INDENT>print("<STR_LIT>")<EOL>pprint.pprint(jobResults["<STR_LIT>"], indent=<NUM_LIT:4>)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>if bestModel is not None:<EOL><INDENT>maxKeyLen = max([len(k) for k in sortedMetricsKeys])<EOL>maxKeyLen = max(maxKeyLen, len(optimizationMetricKey))<EOL>formatStr = "<STR_LIT>" % (maxKeyLen+<NUM_LIT:2>)<EOL>bestMetricValue = list(bestModel.getOptimizationMetrics().values())[<NUM_LIT:0>]<EOL>optimizationMetricName = list(bestModel.getOptimizationMetrics().keys())[<NUM_LIT:0>]<EOL>print()<EOL>print("<STR_LIT>" % (<EOL>optimizationMetricName, maximizeMetric))<EOL>print("<STR_LIT>" % (<EOL>bestModelIterIndex, bestModel, bestModel.getModelDescription()))<EOL>print(formatStr % (optimizationMetricName+"<STR_LIT::>"), bestMetricValue)<EOL>print()<EOL>print("<STR_LIT>"  % totalRecords)<EOL>print()<EOL>print("<STR_LIT>" % totalWallTime)<EOL>hsJobParams = hyperSearchJob.getParams()<EOL><DEDENT>if options["<STR_LIT>"] > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>options["<STR_LIT>"]))<EOL>scoreModelIDDescList.sort()<EOL>scoreModelIDDescList = scoreModelIDDescList[<EOL><NUM_LIT:0>:options["<STR_LIT>"]]<EOL>i = -<NUM_LIT:1><EOL>for (score, modelID, description, paramLabels) in scoreModelIDDescList:<EOL><INDENT>i += <NUM_LIT:1><EOL>outDir = os.path.join(options["<STR_LIT>"], "<STR_LIT>" % (i))<EOL>print("<STR_LIT>" %(modelID, outDir))<EOL>if not os.path.exists(outDir):<EOL><INDENT>os.makedirs(outDir)<EOL><DEDENT>base_description_path = os.path.join(options["<STR_LIT>"],<EOL>"<STR_LIT>")<EOL>base_description_relpath = os.path.relpath(base_description_path,<EOL>start=outDir)<EOL>description = description.replace(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>" % base_description_relpath)<EOL>fd = open(os.path.join(outDir, "<STR_LIT>"), "<STR_LIT:wb>")<EOL>fd.write(description)<EOL>fd.close()<EOL>fd = open(os.path.join(outDir, "<STR_LIT>"), "<STR_LIT:wb>")<EOL>writer = csv.writer(fd)<EOL>colNames = list(paramLabels.keys())<EOL>colNames.sort()<EOL>writer.writerow(colNames)<EOL>row = [paramLabels[x] for x in colNames]<EOL>writer.writerow(row)<EOL>fd.close()<EOL>print("<STR_LIT>")<EOL>mod = imp.load_source("<STR_LIT:description>", os.path.join(outDir,<EOL>"<STR_LIT>"))<EOL>model_description = mod.descriptionInterface.getModelDescription()<EOL>fd = open(os.path.join(outDir, "<STR_LIT>"), "<STR_LIT:wb>")<EOL>fd.write("<STR_LIT>" % (getCopyrightHead(),<EOL>pprint.pformat(model_description)))<EOL>fd.close()<EOL><DEDENT>print()<EOL><DEDENT>reportWriter.finalize()<EOL>return model_description<EOL>

Prints all available results in the given HyperSearch job and emits
        model information to the permutations report csv.

        The job may be completed or still in progress.

        Parameters:
        ----------------------------------------------------------------------
        options:        NupicRunPermutations options dict
        replaceReport:  True to replace existing report csv, if any; False to
                        append to existing report csv, if any
        hyperSearchJob: _HyperSearchJob instance; if None, will get it from saved
                        jobID, if any
        metricsKeys:    sequence of report metrics key names to include in report;
                        if None, will pre-scan all modelInfos to generate a complete
                        list of metrics key names.
        retval:         model parameters


@classmethod<EOL><INDENT>def loadSavedHyperSearchJob(cls, permWorkDir, outputLabel):<DEDENT>

jobID = cls.__loadHyperSearchJobID(permWorkDir=permWorkDir,<EOL>outputLabel=outputLabel)<EOL>searchJob = _HyperSearchJob(nupicJobID=jobID)<EOL>return searchJob<EOL>

Instantiates a _HyperSearchJob instance from info saved in file

        Parameters:
        ----------------------------------------------------------------------
        permWorkDir: Directory path for saved jobID file
        outputLabel: Label string for incorporating into file name for saved jobID
        retval:      _HyperSearchJob instance; raises exception if not found


@classmethod<EOL><INDENT>def __saveHyperSearchJobID(cls, permWorkDir, outputLabel, hyperSearchJob):<DEDENT>

jobID = hyperSearchJob.getJobID()<EOL>filePath = cls.__getHyperSearchJobIDFilePath(permWorkDir=permWorkDir,<EOL>outputLabel=outputLabel)<EOL>if os.path.exists(filePath):<EOL><INDENT>_backupFile(filePath)<EOL><DEDENT>d = dict(hyperSearchJobID = jobID)<EOL>with open(filePath, "<STR_LIT:wb>") as jobIdPickleFile:<EOL><INDENT>pickle.dump(d, jobIdPickleFile)<EOL><DEDENT>

Saves the given _HyperSearchJob instance's jobID to file

        Parameters:
        ----------------------------------------------------------------------
        permWorkDir:   Directory path for saved jobID file
        outputLabel:   Label string for incorporating into file name for saved jobID
        hyperSearchJob: _HyperSearchJob instance
        retval:        nothing


@classmethod<EOL><INDENT>def __loadHyperSearchJobID(cls, permWorkDir, outputLabel):<DEDENT>

filePath = cls.__getHyperSearchJobIDFilePath(permWorkDir=permWorkDir,<EOL>outputLabel=outputLabel)<EOL>jobID = None<EOL>with open(filePath, "<STR_LIT:r>") as jobIdPickleFile:<EOL><INDENT>jobInfo = pickle.load(jobIdPickleFile)<EOL>jobID = jobInfo["<STR_LIT>"]<EOL><DEDENT>return jobID<EOL>

Loads a saved jobID from file

        Parameters:
        ----------------------------------------------------------------------
        permWorkDir:  Directory path for saved jobID file
        outputLabel:  Label string for incorporating into file name for saved jobID
        retval:       HyperSearch jobID; raises exception if not found.


@classmethod<EOL><INDENT>def __getHyperSearchJobIDFilePath(cls, permWorkDir, outputLabel):<DEDENT>

<EOL>basePath = permWorkDir<EOL>filename = "<STR_LIT>" % (outputLabel,)<EOL>filepath = os.path.join(basePath, filename)<EOL>return filepath<EOL>

Returns filepath where to store HyperSearch JobID

        Parameters:
        ----------------------------------------------------------------------
        permWorkDir: Directory path for saved jobID file
        outputLabel: Label string for incorporating into file name for saved jobID
        retval:      Filepath where to store HyperSearch JobID


self.__searchJob = hyperSearchJob<EOL>self.__searchJobID = hyperSearchJob.getJobID()<EOL>self.__sortedMetricsKeys = sorted(metricsKeys)<EOL>self.__outputDirAbsPath = os.path.abspath(outputDirAbsPath)<EOL>self.__outputLabel = outputLabel<EOL>self.__replaceReport = replaceReport<EOL>self.__sortedVariableNames=searchVar<EOL>self.__csvFileObj = None<EOL>self.__reportCSVPath = None<EOL>self.__backupCSVPath = None<EOL>

Parameters:
----------------------------------------------------------------------
hyperSearchJob: _HyperSearchJob instance
metricsKeys:    sequence of report metrics key names to include in report
outputDirAbsPath:
                Directory for creating report CSV file (absolute path)
outputLabel:    A string label to incorporate into report CSV file name
replaceReport:  True to replace existing report csv, if any; False to
                append to existing report csv, if any
retval:         nothing


<EOL>if self.__csvFileObj is None:<EOL><INDENT>self.__openAndInitCSVFile(modelInfo)<EOL><DEDENT>csv = self.__csvFileObj<EOL>print("<STR_LIT>" % (self.__searchJobID), end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>" % (modelInfo.getModelID()), end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>" % (modelInfo.statusAsString()), end='<STR_LIT:U+0020>', file=csv)<EOL>if modelInfo.isFinished():<EOL><INDENT>print("<STR_LIT>" % (modelInfo.getCompletionReason()), end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>if not modelInfo.isWaitingToStart():<EOL><INDENT>print("<STR_LIT>" % (modelInfo.getStartTime()), end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>if modelInfo.isFinished():<EOL><INDENT>dateFormat = "<STR_LIT>"<EOL>startTime = modelInfo.getStartTime()<EOL>endTime = modelInfo.getEndTime()<EOL>print("<STR_LIT>" % endTime, end='<STR_LIT:U+0020>', file=csv)<EOL>st = datetime.strptime(startTime, dateFormat)<EOL>et = datetime.strptime(endTime, dateFormat)<EOL>print("<STR_LIT>" % (str((et - st).seconds)), end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>print("<STR_LIT>" % str(modelInfo.getModelDescription()), end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>" % str(modelInfo.getNumRecords()), end='<STR_LIT:U+0020>', file=csv)<EOL>paramLabelsDict = modelInfo.getParamLabels()<EOL>for key in self.__sortedVariableNames:<EOL><INDENT>if key in paramLabelsDict:<EOL><INDENT>print("<STR_LIT>" % (paramLabelsDict[key]), end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT><DEDENT>metrics = modelInfo.getReportMetrics()<EOL>for key in self.__sortedMetricsKeys:<EOL><INDENT>value = metrics.get(key, "<STR_LIT>")<EOL>value = str(value)<EOL>value = value.replace("<STR_LIT:\n>", "<STR_LIT:U+0020>")<EOL>print("<STR_LIT>" % (value), end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>print(file=csv)<EOL>

Emit model info to csv file

        Parameters:
        ----------------------------------------------------------------------
        modelInfo:      _NupicModelInfo instance
        retval:         nothing


if self.__csvFileObj is not None:<EOL><INDENT>self.__csvFileObj.close()<EOL>self.__csvFileObj = None<EOL>print("<STR_LIT>" % (self.__reportCSVPath,))<EOL>if self.__backupCSVPath:<EOL><INDENT>print("<STR_LIT>" %(self.__backupCSVPath,))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>


<EOL>basePath = self.__outputDirAbsPath<EOL>reportCSVName = "<STR_LIT>" % (self.__outputLabel,)<EOL>reportCSVPath = self.__reportCSVPath = os.path.join(basePath, reportCSVName)<EOL>backupCSVPath = None<EOL>if os.path.exists(reportCSVPath):<EOL><INDENT>backupCSVPath = self.__backupCSVPath = _backupFile(reportCSVPath)<EOL><DEDENT>if self.__replaceReport:<EOL><INDENT>mode = "<STR_LIT:w>"<EOL><DEDENT>else:<EOL><INDENT>mode = "<STR_LIT:a>"<EOL><DEDENT>csv = self.__csvFileObj = open(reportCSVPath, mode)<EOL>if not self.__replaceReport and backupCSVPath:<EOL><INDENT>print(file=csv)<EOL>print(file=csv)<EOL><DEDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>print("<STR_LIT>", end='<STR_LIT:U+0020>', file=csv)<EOL>for key in self.__sortedVariableNames:<EOL><INDENT>print("<STR_LIT>" % key, end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>for key in self.__sortedMetricsKeys:<EOL><INDENT>print("<STR_LIT>" % key, end='<STR_LIT:U+0020>', file=csv)<EOL><DEDENT>print(file=csv)<EOL>

- Backs up old report csv file;
- opens the report csv file in append or overwrite mode (per
  self.__replaceReport);
- emits column fields;
- sets up self.__sortedVariableNames, self.__csvFileObj,
  self.__backupCSVPath, and self.__reportCSVPath

Parameters:
----------------------------------------------------------------------
modelInfo:      First _NupicModelInfo instance passed to emit()
retval:         nothing


self.__nupicJobID = nupicJobID<EOL>jobInfo = _clientJobsDB().jobInfo(nupicJobID)<EOL>assert jobInfo is not None, "<STR_LIT>" % nupicJobID<EOL>assert jobInfo.jobId == nupicJobID, "<STR_LIT>" % (jobInfo.jobId, nupicJobID)<EOL>_emit(Verbosity.DEBUG, "<STR_LIT>" % pprint.pformat(jobInfo, indent=<NUM_LIT:4>))<EOL>if jobInfo.params is not None:<EOL><INDENT>self.__params = json.loads(jobInfo.params)<EOL><DEDENT>else:<EOL><INDENT>self.__params = None<EOL><DEDENT>


return "<STR_LIT>" % (self.__class__.__name__, self.__nupicJobID)<EOL>


jobInfo = self.JobStatus(self.__nupicJobID, workers)<EOL>return jobInfo<EOL>

Parameters:
----------------------------------------------------------------------
workers:  If this job was launched outside of the nupic job engine, then this
           is an array of subprocess Popen instances, one for each worker
retval:         _NupicJob.JobStatus instance


return self.__nupicJobID<EOL>

Semi-private method for retrieving the jobId

        Parameters:
        ----------------------------------------------------------------------
        retval:         Nupic Client JobID of this _NupicJob instance


return self.__params<EOL>

Semi-private method for retrieving the job-specific params

        Parameters:
        ----------------------------------------------------------------------
        retval:         Job params dict corresponding to the JSON params value
                        returned by ClientJobsDAO.jobInfo()


self.__reason = reason<EOL>


super(_HyperSearchJob, self).__init__(nupicJobID)<EOL>self.__expectedNumModels = None<EOL>


jobID = self.getJobID()<EOL>modelCounterPairs = _clientJobsDB().modelsGetUpdateCounters(jobID)<EOL>modelIDs = tuple(x[<NUM_LIT:0>] for x in modelCounterPairs)<EOL>return modelIDs<EOL>

Queuries DB for model IDs of all currently instantiated models
        associated with this HyperSearch job.

        See also: _iterModels()

        Parameters:
        ----------------------------------------------------------------------
        retval:         A sequence of Nupic modelIDs


return self.__expectedNumModels<EOL>

Returns:  the total number of expected models if known, -1 if it can't
        be determined.

        NOTE: this can take a LONG time to complete for HyperSearches with a huge
              number of possible permutations.

        Parameters:
        ----------------------------------------------------------------------
        searchMethod:   "v2" is the only method currently supported
        retval:         The total number of expected models, if known; -1 if unknown


@classmethod<EOL><INDENT>def makeSearchJobParamsDict(cls, options, forRunning=False):<DEDENT>

if options["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>hsVersion = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % options["<STR_LIT>"])<EOL><DEDENT>maxModels = options["<STR_LIT>"]<EOL>if options["<STR_LIT:action>"] == "<STR_LIT>" and maxModels is None:<EOL><INDENT>maxModels = <NUM_LIT:1><EOL><DEDENT>useTerminators = options["<STR_LIT>"]<EOL>if useTerminators is None:<EOL><INDENT>params = {<EOL>"<STR_LIT>":          hsVersion,<EOL>"<STR_LIT>":          maxModels,<EOL>}<EOL><DEDENT>else:<EOL><INDENT>params = {<EOL>"<STR_LIT>":          hsVersion,<EOL>"<STR_LIT>":     useTerminators,<EOL>"<STR_LIT>":          maxModels,<EOL>}<EOL><DEDENT>if forRunning:<EOL><INDENT>params["<STR_LIT>"] = str(uuid.uuid1())<EOL><DEDENT>if options["<STR_LIT>"]:<EOL><INDENT>params["<STR_LIT>"] = options["<STR_LIT>"]<EOL><DEDENT>elif options["<STR_LIT>"]:<EOL><INDENT>params["<STR_LIT:description>"] = options["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>with open(options["<STR_LIT>"], mode="<STR_LIT:r>") as fp:<EOL><INDENT>params["<STR_LIT:description>"] = json.load(fp)<EOL><DEDENT><DEDENT>return params<EOL>

Constructs a dictionary of HyperSearch parameters suitable for converting
        to json and passing as the params argument to ClientJobsDAO.jobInsert()
        Parameters:
        ----------------------------------------------------------------------
        options:        NupicRunPermutations options dict
        forRunning:     True if the params are for running a Hypersearch job; False
                        if params are for introspection only.

        retval:         A dictionary of HyperSearch parameters for
                        ClientJobsDAO.jobInsert()


if searchJobParams["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>search = HypersearchV2(searchParams=searchJobParams)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(searchJobParams["<STR_LIT>"]))<EOL><DEDENT>info = search.getOptimizationMetricInfo()<EOL>return info<EOL>

Retrives the optimization key name and optimization function.

        Parameters:
        ---------------------------------------------------------
        searchJobParams:
                        Parameter for passing as the searchParams arg to
                        Hypersearch constructor.
        retval:       (optimizationMetricKey, maximize)
                      optimizationMetricKey: which report key to optimize for
                      maximize: True if we should try and maximize the optimizeKey
                        metric. False if we should minimize it.


self.__rawInfo = rawInfo<EOL>self.__cachedResults = None<EOL>assert self.__rawInfo.params is not None<EOL>self.__cachedParams = None<EOL>

Parameters:
----------------------------------------------------------------------
rawInfo:        A single model information element as returned by
                ClientJobsDAO.modelsInfo()
retval:         nothing.


return self.__rawInfo.modelId<EOL>


return "<STR_LIT:%s>" % self.__rawInfo.status<EOL>


params = self.__unwrapParams()<EOL>if "<STR_LIT>" in params:<EOL><INDENT>return params["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>paramSettings = self.getParamLabels()<EOL>items = []<EOL>for key, value in list(paramSettings.items()):<EOL><INDENT>items.append("<STR_LIT>" % (key, value))<EOL><DEDENT>return "<STR_LIT:.>".join(items)<EOL><DEDENT>


return self.__rawInfo.genDescription<EOL>


return self.__rawInfo.numRecords<EOL>


params = self.__unwrapParams()<EOL>if "<STR_LIT>" in params:<EOL><INDENT>retval = dict()<EOL>queue = [(pair, retval) for pair in<EOL>params["<STR_LIT>"]["<STR_LIT>"].items()]<EOL>while len(queue) > <NUM_LIT:0>:<EOL><INDENT>pair, output = queue.pop()<EOL>k, v = pair<EOL>if ("<STR_LIT>" in v and "<STR_LIT>" in v and<EOL>"<STR_LIT>" in v):<EOL><INDENT>output[k] = v["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>if k not in output:<EOL><INDENT>output[k] = dict()<EOL><DEDENT>queue.extend((pair, output[k]) for pair in v.items())<EOL><DEDENT><DEDENT>return retval<EOL><DEDENT>

Parameters:
----------------------------------------------------------------------
retval:         a dictionary of model parameter labels. For each entry
                the key is the name of the parameter and the value
                is the value chosen for it.


if self.__cachedParams is None:<EOL><INDENT>self.__cachedParams = json.loads(self.__rawInfo.params)<EOL>assert self.__cachedParams is not None,"<STR_LIT>" % self.__rawInfo.params<EOL><DEDENT>return self.__cachedParams<EOL>

Unwraps self.__rawInfo.params into the equivalent python dictionary
        and caches it in self.__cachedParams. Returns the unwrapped params

        Parameters:
        ----------------------------------------------------------------------
        retval:         Model params dictionary as correpsonding to the json
                        as returned in ClientJobsDAO.modelsInfo()[x].params


return self.__unwrapResults().reportMetrics<EOL>

Retrives a dictionary of metrics designated for report
        Parameters:
        ----------------------------------------------------------------------
        retval: a dictionary of metrics that were collected for the model or
                an empty dictionary if there aren't any.


return self.__unwrapResults().optimizationMetrics<EOL>

Retrives a dictionary of metrics designagted for optimization
        Parameters:
        ----------------------------------------------------------------------
        retval:         a dictionary of optimization metrics that were collected
                        for the model or an empty dictionary if there aren't any.


result = self.getReportMetrics()<EOL>result.update(self.getOptimizationMetrics())<EOL>return result<EOL>

Retrives a dictionary of metrics that combines all report and
        optimization metrics

        Parameters:
        ----------------------------------------------------------------------
        retval:         a dictionary of optimization metrics that were collected
                        for the model; an empty dictionary if there aren't any.


if self.__cachedResults is None:<EOL><INDENT>if self.__rawInfo.results is not None:<EOL><INDENT>resultList = json.loads(self.__rawInfo.results)<EOL>assert len(resultList) == <NUM_LIT:2>,"<STR_LIT>" % (<EOL>len(resultList), resultList)<EOL>self.__cachedResults = self.ModelResults(<EOL>reportMetrics=resultList[<NUM_LIT:0>],<EOL>optimizationMetrics=resultList[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>self.__cachedResults = self.ModelResults(<EOL>reportMetrics={},<EOL>optimizationMetrics={})<EOL><DEDENT><DEDENT>return self.__cachedResults<EOL>

Unwraps self.__rawInfo.results and caches it in self.__cachedResults;
        Returns the unwrapped params

        Parameters:
        ----------------------------------------------------------------------
        retval:         ModelResults namedtuple instance


waiting = (self.__rawInfo.status == self.__nupicModelStatus_notStarted)<EOL>return waiting<EOL>


running = (self.__rawInfo.status == self.__nupicModelStatus_running)<EOL>return running<EOL>


finished = (self.__rawInfo.status == self.__nupicModelStatus_completed)<EOL>return finished<EOL>

Parameters:
----------------------------------------------------------------------
retval:         True if the model's processing has completed (either with
                success or failure).


assert self.isFinished(), "<STR_LIT>" % self<EOL>return _ModelCompletionReason(self.__rawInfo.completionReason)<EOL>

Returns _ModelCompletionReason.

        NOTE: it's an error to call this method if isFinished() would return False.

        Parameters:
        ----------------------------------------------------------------------
        retval:         _ModelCompletionReason instance


assert self.isFinished(), "<STR_LIT>" % self<EOL>return self.__rawInfo.completionMsg<EOL>

Returns model completion message.

        NOTE: it's an error to call this method if isFinished() would return False.

        Parameters:
        ----------------------------------------------------------------------
        retval:         completion message


assert not self.isWaitingToStart(), "<STR_LIT>" % self<EOL>return "<STR_LIT:%s>" % self.__rawInfo.startTime<EOL>

Returns model evaluation start time.

        NOTE: it's an error to call this method if isWaitingToStart() would
        return True.

        Parameters:
        ----------------------------------------------------------------------
        retval:         model evaluation start time


assert self.isFinished(), "<STR_LIT>" % self<EOL>return "<STR_LIT:%s>" % self.__rawInfo.endTime<EOL>

Returns mode evaluation end time.

        NOTE: it's an error to call this method if isFinished() would return False.

        Parameters:
        ----------------------------------------------------------------------
        retval:         model evaluation end time


parser = OptionParser(helpString)<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:int>", default=None,<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:str>", default=None,<EOL>help=("<STR_LIT>"<EOL>"<STR_LIT>"))<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:str>", default=None,<EOL>help=("<STR_LIT>"<EOL>"<STR_LIT>"))<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", default=None,<EOL>help="<STR_LIT>""<STR_LIT>""<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store_true>", default=False,<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store_true>", default=False,<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:int>", default=None,<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>(options, args) = parser.parse_args(argv[<NUM_LIT:1>:])<EOL>if len(args) != <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(args))<EOL><DEDENT>if (options.jobID and options.params):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if (options.jobID is None and options.params is None):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>initLogging(verbose=True)<EOL>hst = HypersearchWorker(options, argv[<NUM_LIT:1>:])<EOL>if options.params is None:<EOL><INDENT>try:<EOL><INDENT>jobID = hst.run()<EOL><DEDENT>except Exception as e:<EOL><INDENT>jobID = options.jobID<EOL>msg = io.StringIO()<EOL>print("<STR_LIT>" %(ErrorCodes.hypersearchLogicErr, e), file=msg)<EOL>traceback.print_exc(None, msg)<EOL>completionReason = ClientJobsDAO.CMPL_REASON_ERROR<EOL>completionMsg = msg.getvalue()<EOL>hst.logger.error(completionMsg)<EOL>jobsDAO = ClientJobsDAO.get()<EOL>workerCmpReason = jobsDAO.jobGetFields(options.jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:<EOL><INDENT>jobsDAO.jobSetFields(options.jobID, fields=dict(<EOL>cancel=True,<EOL>workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,<EOL>workerCompletionMsg = completionMsg),<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>jobID = None<EOL>completionReason = ClientJobsDAO.CMPL_REASON_SUCCESS<EOL>completionMsg = "<STR_LIT>"<EOL>try:<EOL><INDENT>jobID = hst.run()<EOL><DEDENT>except Exception as e:<EOL><INDENT>jobID = hst._options.jobID<EOL>completionReason = ClientJobsDAO.CMPL_REASON_ERROR<EOL>completionMsg = "<STR_LIT>" % (e,)<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>if jobID is not None:<EOL><INDENT>cjDAO = ClientJobsDAO.get()<EOL>cjDAO.jobSetCompleted(jobID=jobID,<EOL>completionReason=completionReason,<EOL>completionMsg=completionMsg)<EOL><DEDENT><DEDENT><DEDENT>return jobID<EOL>

The main function of the HypersearchWorker script. This parses the command
line arguments, instantiates a HypersearchWorker instance, and then
runs it.

Parameters:
----------------------------------------------------------------------
retval:     jobID of the job we ran. This is used by unit test code
              when calling this working using the --params command
              line option (which tells this worker to insert the job
              itself).


def __init__(self, options, cmdLineArgs):

<EOL>self._options = options<EOL>self.logger = logging.getLogger("<STR_LIT:.>".join(<EOL>['<STR_LIT>', self.__class__.__name__]))<EOL>if options.logLevel is not None:<EOL><INDENT>self.logger.setLevel(options.logLevel)<EOL><DEDENT>self.logger.info("<STR_LIT>" %<EOL>str(cmdLineArgs))<EOL>self.logger.debug("<STR_LIT>" % (pprint.pformat(os.environ)))<EOL>random.seed(<NUM_LIT>)<EOL>self._hs = None<EOL>self._modelIDCtrDict = dict()<EOL>self._modelIDCtrList = []<EOL>self._modelIDSet = set()<EOL>self._workerID = None<EOL>

Instantiate the Hypersearch worker

        Parameters:
        ---------------------------------------------------------------------
        options:      The command line options. See the main() method for a
                        description of these options
        cmdLineArgs:  Copy of the command line arguments, so we can place them
                        in the log


<EOL>curModelIDCtrList = cjDAO.modelsGetUpdateCounters(self._options.jobID)<EOL>if len(curModelIDCtrList) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>self.logger.debug("<STR_LIT>"% (str(curModelIDCtrList)))<EOL>self.logger.debug("<STR_LIT>"% (str(self._modelIDCtrList)))<EOL>curModelIDCtrList = sorted(curModelIDCtrList)<EOL>numItems = len(curModelIDCtrList)<EOL>changedEntries = [x for x in zip(range(numItems), curModelIDCtrList,<EOL>self._modelIDCtrList) if x[<NUM_LIT:1>][<NUM_LIT:1>] != x[<NUM_LIT:2>][<NUM_LIT:1>]]<EOL>if len(changedEntries) > <NUM_LIT:0>:<EOL><INDENT>self.logger.debug("<STR_LIT>", str(changedEntries))<EOL>for entry in changedEntries:<EOL><INDENT>(idx, (modelID, curCtr), (_, oldCtr)) = entry<EOL>self._modelIDCtrDict[modelID] = curCtr<EOL>assert (self._modelIDCtrList[idx][<NUM_LIT:0>] == modelID)<EOL>assert (curCtr != oldCtr)<EOL>self._modelIDCtrList[idx][<NUM_LIT:1>] = curCtr<EOL><DEDENT>changedModelIDs = [x[<NUM_LIT:1>][<NUM_LIT:0>] for x in changedEntries]<EOL>modelResults = cjDAO.modelsGetResultAndStatus(changedModelIDs)<EOL>for mResult in modelResults:<EOL><INDENT>results = mResult.results<EOL>if results is not None:<EOL><INDENT>results = json.loads(results)<EOL><DEDENT>self._hs.recordModelProgress(modelID=mResult.modelId,<EOL>modelParams = None,<EOL>modelParamsHash = mResult.engParamsHash,<EOL>results = results,<EOL>completed = (mResult.status == cjDAO.STATUS_COMPLETED),<EOL>completionReason = mResult.completionReason,<EOL>matured = mResult.engMatured,<EOL>numRecords = mResult.numRecords)<EOL><DEDENT><DEDENT>curModelIDSet = set([x[<NUM_LIT:0>] for x in curModelIDCtrList])<EOL>newModelIDs = curModelIDSet.difference(self._modelIDSet)<EOL>if len(newModelIDs) > <NUM_LIT:0>:<EOL><INDENT>self._modelIDSet.update(newModelIDs)<EOL>curModelIDCtrDict = dict(curModelIDCtrList)<EOL>modelInfos = cjDAO.modelsGetResultAndStatus(newModelIDs)<EOL>modelInfos.sort()<EOL>modelParamsAndHashs = cjDAO.modelsGetParams(newModelIDs)<EOL>modelParamsAndHashs.sort()<EOL>for (mResult, mParamsAndHash) in zip(modelInfos,<EOL>modelParamsAndHashs):<EOL><INDENT>modelID = mResult.modelId<EOL>assert (modelID == mParamsAndHash.modelId)<EOL>self._modelIDCtrDict[modelID] = curModelIDCtrDict[modelID]<EOL>self._modelIDCtrList.append([modelID, curModelIDCtrDict[modelID]])<EOL>results = mResult.results<EOL>if results is not None:<EOL><INDENT>results = json.loads(mResult.results)<EOL><DEDENT>self._hs.recordModelProgress(modelID = modelID,<EOL>modelParams = json.loads(mParamsAndHash.params),<EOL>modelParamsHash = mParamsAndHash.engParamsHash,<EOL>results = results,<EOL>completed = (mResult.status == cjDAO.STATUS_COMPLETED),<EOL>completionReason = (mResult.completionReason),<EOL>matured = mResult.engMatured,<EOL>numRecords = mResult.numRecords)<EOL><DEDENT>self._modelIDCtrList.sort()<EOL><DEDENT>

For all models that modified their results since last time this method
        was called, send their latest results to the Hypersearch implementation.


<EOL>options = self._options<EOL>self.logger.info("<STR_LIT>")<EOL>cjDAO = ClientJobsDAO.get()<EOL>self._workerID = cjDAO.getConnectionID()<EOL>if options.clearModels:<EOL><INDENT>cjDAO.modelsClearAll()<EOL><DEDENT>if options.params is not None:<EOL><INDENT>options.jobID = cjDAO.jobInsert(client='<STR_LIT>', cmdLine="<STR_LIT>",<EOL>params=options.params, alreadyRunning=True,<EOL>minimumWorkers=<NUM_LIT:1>, maximumWorkers=<NUM_LIT:1>,<EOL>jobType = cjDAO.JOB_TYPE_HS)<EOL><DEDENT>if options.workerID is not None:<EOL><INDENT>wID = options.workerID<EOL><DEDENT>else:<EOL><INDENT>wID = self._workerID<EOL><DEDENT>buildID = Configuration.get('<STR_LIT>', '<STR_LIT>')<EOL>logPrefix = '<STR_LIT>' %(buildID, wID, options.jobID)<EOL>ExtendedLogger.setLogPrefix(logPrefix)<EOL>if options.resetJobStatus:<EOL><INDENT>cjDAO.jobSetFields(options.jobID,<EOL>fields={'<STR_LIT>': ClientJobsDAO.CMPL_REASON_SUCCESS,<EOL>'<STR_LIT>': False,<EOL>},<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL><DEDENT>jobInfo = cjDAO.jobInfo(options.jobID)<EOL>self.logger.info("<STR_LIT>" % (str(clippedObj(jobInfo))))<EOL>jobParams = json.loads(jobInfo.params)<EOL>jsonSchemaPath = os.path.join(os.path.dirname(__file__),<EOL>"<STR_LIT>",<EOL>"<STR_LIT>")<EOL>validate(jobParams, schemaPath=jsonSchemaPath)<EOL>hsVersion = jobParams.get('<STR_LIT>', None)<EOL>if hsVersion == '<STR_LIT>':<EOL><INDENT>self._hs = HypersearchV2(searchParams=jobParams, workerID=self._workerID,<EOL>cjDAO=cjDAO, jobID=options.jobID, logLevel=options.logLevel)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>"% (hsVersion))<EOL><DEDENT>try:<EOL><INDENT>exit = False<EOL>numModelsTotal = <NUM_LIT:0><EOL>print("<STR_LIT>", file=sys.stderr)<EOL>while not exit:<EOL><INDENT>batchSize = <NUM_LIT:10>              <EOL>modelIDToRun = None<EOL>while modelIDToRun is None:<EOL><INDENT>if options.modelID is None:<EOL><INDENT>self._processUpdatedModels(cjDAO)<EOL>(exit, newModels) = self._hs.createModels(numModels = batchSize)<EOL>if exit:<EOL><INDENT>break<EOL><DEDENT>if len(newModels) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>for (modelParams, modelParamsHash, particleHash) in newModels:<EOL><INDENT>jsonModelParams = json.dumps(modelParams)<EOL>(modelID, ours) = cjDAO.modelInsertAndStart(options.jobID,<EOL>jsonModelParams, modelParamsHash, particleHash)<EOL>if not ours:<EOL><INDENT>mParamsAndHash = cjDAO.modelsGetParams([modelID])[<NUM_LIT:0>]<EOL>mResult = cjDAO.modelsGetResultAndStatus([modelID])[<NUM_LIT:0>]<EOL>results = mResult.results<EOL>if results is not None:<EOL><INDENT>results = json.loads(results)<EOL><DEDENT>modelParams = json.loads(mParamsAndHash.params)<EOL>particleHash = cjDAO.modelsGetFields(modelID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>particleInst = "<STR_LIT>" % (<EOL>modelParams['<STR_LIT>']['<STR_LIT:id>'],<EOL>modelParams['<STR_LIT>']['<STR_LIT>'])<EOL>self.logger.info("<STR_LIT>""<STR_LIT>""<STR_LIT>", modelID,<EOL>mParamsAndHash.engParamsHash.encode('<STR_LIT>'),<EOL>particleHash.encode('<STR_LIT>'), particleInst)<EOL>self._hs.recordModelProgress(modelID = modelID,<EOL>modelParams = modelParams,<EOL>modelParamsHash = mParamsAndHash.engParamsHash,<EOL>results = results,<EOL>completed = (mResult.status == cjDAO.STATUS_COMPLETED),<EOL>completionReason = mResult.completionReason,<EOL>matured = mResult.engMatured,<EOL>numRecords = mResult.numRecords)<EOL><DEDENT>else:<EOL><INDENT>modelIDToRun = modelID<EOL>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>modelIDToRun = int(options.modelID)<EOL>mParamsAndHash = cjDAO.modelsGetParams([modelIDToRun])[<NUM_LIT:0>]<EOL>modelParams = json.loads(mParamsAndHash.params)<EOL>modelParamsHash = mParamsAndHash.engParamsHash<EOL>cjDAO.modelSetFields(modelIDToRun,<EOL>dict(engWorkerConnId=self._workerID))<EOL>if False:<EOL><INDENT>for attempt in range(<NUM_LIT:1000>):<EOL><INDENT>paramsHash = hashlib.md5("<STR_LIT>" % (modelIDToRun,<EOL>attempt)).digest()<EOL>particleHash = hashlib.md5("<STR_LIT>" % (modelIDToRun,<EOL>attempt)).digest()<EOL>try:<EOL><INDENT>cjDAO.modelSetFields(modelIDToRun,<EOL>dict(engParamsHash=paramsHash,<EOL>engParticleHash=particleHash))<EOL>success = True<EOL><DEDENT>except:<EOL><INDENT>success = False<EOL><DEDENT>if success:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not success:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>(modelIDToRun, ours) = cjDAO.modelInsertAndStart(options.jobID,<EOL>mParamsAndHash.params, modelParamsHash)<EOL><DEDENT><DEDENT><DEDENT>if exit:<EOL><INDENT>break<EOL><DEDENT>self.logger.info("<STR_LIT>",<EOL>modelIDToRun, modelParamsHash.encode('<STR_LIT>'), modelParams)<EOL>persistentJobGUID = jobParams['<STR_LIT>']<EOL>assert persistentJobGUID, "<STR_LIT>" % (persistentJobGUID,)<EOL>modelCheckpointGUID = jobInfo.client + "<STR_LIT:_>" + persistentJobGUID + (<EOL>'<STR_LIT:_>' + str(modelIDToRun))<EOL>self._hs.runModel(modelID=modelIDToRun, jobID = options.jobID,<EOL>modelParams=modelParams, modelParamsHash=modelParamsHash,<EOL>jobsDAO=cjDAO, modelCheckpointGUID=modelCheckpointGUID)<EOL>numModelsTotal += <NUM_LIT:1><EOL>self.logger.info("<STR_LIT>",<EOL>modelIDToRun, numModelsTotal)<EOL>print("<STR_LIT>" %(numModelsTotal), file=sys.stderr)<EOL>print("<STR_LIT>", file=sys.stderr)<EOL>if options.modelID is not None:<EOL><INDENT>exit = True<EOL><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>self._hs.close()<EOL><DEDENT>self.logger.info("<STR_LIT>" % (numModelsTotal))<EOL>print("<STR_LIT>" % (numModelsTotal), file=sys.stderr)<EOL>return options.jobID<EOL>

Run this worker.

        Parameters:
        ----------------------------------------------------------------------
        retval:     jobID of the job we ran. This is used by unit test code
                      when calling this working using the --params command
                      line option (which tells this worker to insert the job
                      itself).


def createAndStartSwarm(client, clientInfo="<STR_LIT>", clientKey="<STR_LIT>", params="<STR_LIT>",<EOL>minimumWorkers=None, maximumWorkers=None,<EOL>alreadyRunning=False):

if minimumWorkers is None:<EOL><INDENT>minimumWorkers = Configuration.getInt(<EOL>"<STR_LIT>")<EOL><DEDENT>if maximumWorkers is None:<EOL><INDENT>maximumWorkers = Configuration.getInt(<EOL>"<STR_LIT>")<EOL><DEDENT>return ClientJobsDAO.get().jobInsert(<EOL>client=client,<EOL>cmdLine="<STR_LIT>",<EOL>clientInfo=clientInfo,<EOL>clientKey=clientKey,<EOL>alreadyRunning=alreadyRunning,<EOL>params=params,<EOL>minimumWorkers=minimumWorkers,<EOL>maximumWorkers=maximumWorkers,<EOL>jobType=ClientJobsDAO.JOB_TYPE_HS)<EOL>

Create and start a swarm job.

    Args:
      client - A string identifying the calling client. There is a small limit
          for the length of the value. See ClientJobsDAO.CLIENT_MAX_LEN.
      clientInfo - JSON encoded dict of client specific information.
      clientKey - Foreign key. Limited in length, see ClientJobsDAO._initTables.
      params - JSON encoded dict of the parameters for the job. This can be
          fetched out of the database by the worker processes based on the jobID.
      minimumWorkers - The minimum workers to allocate to the swarm. Set to None
          to use the default.
      maximumWorkers - The maximum workers to allocate to the swarm. Set to None
          to use the swarm default. Set to 0 to use the maximum scheduler value.
      alreadyRunning - Insert a job record for an already running process. Used
          for testing.


<EOL>cjDAO = ClientJobsDAO.get()<EOL>(jobID, description) = cjDAO.modelsGetFields(<EOL>modelID,<EOL>["<STR_LIT>", "<STR_LIT>"])<EOL>(baseDescription,) = cjDAO.jobGetFields(jobID, ["<STR_LIT>"])<EOL>descriptionDirectory = tempfile.mkdtemp()<EOL>try:<EOL><INDENT>baseDescriptionFilePath = os.path.join(descriptionDirectory, "<STR_LIT>")<EOL>with open(baseDescriptionFilePath, mode="<STR_LIT:wb>") as f:<EOL><INDENT>f.write(baseDescription)<EOL><DEDENT>descriptionFilePath = os.path.join(descriptionDirectory, "<STR_LIT>")<EOL>with open(descriptionFilePath, mode="<STR_LIT:wb>") as f:<EOL><INDENT>f.write(description)<EOL><DEDENT>expIface = helpers.getExperimentDescriptionInterfaceFromModule(<EOL>helpers.loadExperimentDescriptionScriptFromDir(descriptionDirectory))<EOL>return json.dumps(<EOL>dict(<EOL>modelConfig=expIface.getModelDescription(),<EOL>inferenceArgs=expIface.getModelControl().get("<STR_LIT>", None)))<EOL><DEDENT>finally:<EOL><INDENT>shutil.rmtree(descriptionDirectory, ignore_errors=True)<EOL><DEDENT>

Retrieve the Engine-level model params from a Swarm model

    Args:
      modelID - Engine-level model ID of the Swarm model

    Returns:
      JSON-encoded string containing Model Params


return "<STR_LIT>" % (errorString, usageString)<EOL>

Combines an error string and usage string into a regular format, so they
    all look consistent.


print("<STR_LIT>")<EOL>print((json.dumps(_getExperimentDescriptionSchema(), indent=_INDENT_STEP*<NUM_LIT:2>)))<EOL>print("<STR_LIT>")<EOL>return<EOL>

Displays command schema to stdout and exit program


def _handleDescriptionOption(cmdArgStr, outDir, usageStr, hsVersion,<EOL>claDescriptionTemplateFile):

<EOL>try:<EOL><INDENT>args = json.loads(cmdArgStr)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise _InvalidCommandArgException(<EOL>_makeUsageErrorStr(<EOL>("<STR_LIT>" +"<STR_LIT>") % (str(e), cmdArgStr), usageStr))<EOL><DEDENT>filesDescription = _generateExperiment(args, outDir, hsVersion=hsVersion,<EOL>claDescriptionTemplateFile = claDescriptionTemplateFile)<EOL>pprint.pprint(filesDescription)<EOL>return<EOL>

Parses and validates the --description option args and executes the
request

Parameters:
-----------------------------------------------------------------------
cmdArgStr:  JSON string compatible with _gExperimentDescriptionSchema
outDir:     where to place generated experiment files
usageStr:   program usage string
hsVersion:  which version of hypersearch permutations file to generate, can
              be 'v1' or 'v2'
claDescriptionTemplateFile: Filename containing the template description
retval:     nothing


def _handleDescriptionFromFileOption(filename, outDir, usageStr, hsVersion,<EOL>claDescriptionTemplateFile):

try:<EOL><INDENT>fileHandle = open(filename, '<STR_LIT:r>')<EOL>JSONStringFromFile = fileHandle.read().splitlines()<EOL>JSONStringFromFile = '<STR_LIT>'.join(JSONStringFromFile)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise _InvalidCommandArgException(<EOL>_makeUsageErrorStr(<EOL>("<STR_LIT>" +"<STR_LIT>") % (str(e), filename), usageStr))<EOL><DEDENT>_handleDescriptionOption(JSONStringFromFile, outDir, usageStr,<EOL>hsVersion=hsVersion,<EOL>claDescriptionTemplateFile = claDescriptionTemplateFile)<EOL>return<EOL>

Parses and validates the --descriptionFromFile option and executes the
request

Parameters:
-----------------------------------------------------------------------
filename:   File from which we'll extract description JSON
outDir:     where to place generated experiment files
usageStr:   program usage string
hsVersion:  which version of hypersearch permutations file to generate, can
              be 'v1' or 'v2'
claDescriptionTemplateFile: Filename containing the template description
retval:     nothing


xInt = int(round(x))<EOL>return (abs(x - xInt) < precision * x, xInt)<EOL>

Return (isInt, intValue) for a given floating point number.

Parameters:
----------------------------------------------------------------------
x:  floating point number to evaluate
precision: desired precision
retval:   (isInt, intValue)
          isInt: True if x is close enough to an integer value
          intValue: x as an integer


return type(obj) in (str,)<EOL>

returns whether or not the object is a string


assert _isString(string)<EOL>return pprint.pformat(string)<EOL>

string:   input string (ascii or unicode)

Returns:  a quoted string with characters that are represented in python via
          escape sequences converted to those escape sequences


def _indentLines(str, indentLevels = <NUM_LIT:1>, indentFirstLine=True):

indent = _ONE_INDENT * indentLevels<EOL>lines = str.splitlines(True)<EOL>result = '<STR_LIT>'<EOL>if len(lines) > <NUM_LIT:0> and not indentFirstLine:<EOL><INDENT>first = <NUM_LIT:1><EOL>result += lines[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>first = <NUM_LIT:0><EOL><DEDENT>for line in lines[first:]:<EOL><INDENT>result += indent + line<EOL><DEDENT>return result<EOL>

Indent all lines in the given string

    str:          input string
    indentLevels: number of levels of indentation to apply
    indentFirstLine: if False, the 1st line will not be indented

    Returns:      The result string with all lines indented


if fieldType == '<STR_LIT:string>':<EOL><INDENT>return True<EOL><DEDENT>if fieldType == '<STR_LIT:int>' or fieldType=='<STR_LIT:float>':<EOL><INDENT>return False<EOL><DEDENT>

Prediction function for determining whether a function is a categorical
        variable or a scalar variable.  Mainly used for determining the appropriate
        metrics.


def _generateMetricSpecString(inferenceElement, metric,<EOL>params=None, field=None,<EOL>returnLabel=False):

metricSpecArgs = dict(metric=metric,<EOL>field=field,<EOL>params=params,<EOL>inferenceElement=inferenceElement)<EOL>metricSpecAsString = "<STR_LIT>" %'<STR_LIT:U+002CU+0020>'.join(['<STR_LIT>' % (item[<NUM_LIT:0>],item[<NUM_LIT:1>])<EOL>for item in metricSpecArgs.items()])<EOL>if not returnLabel:<EOL><INDENT>return metricSpecAsString<EOL><DEDENT>spec = MetricSpec(**metricSpecArgs)<EOL>metricLabel = spec.getLabel()<EOL>return metricSpecAsString, metricLabel<EOL>

Generates the string representation of a MetricSpec object, and returns
    the metric key associated with the metric.

    Parameters:
    -----------------------------------------------------------------------
    inferenceElement:
      An InferenceElement value that indicates which part of the inference this
      metric is computed on

    metric:
      The type of the metric being computed (e.g. aae, avg_error)

    params:
      A dictionary of parameters for the metric. The keys are the parameter names
      and the values should be the parameter values (e.g. window=200)

    field:
      The name of the field for which this metric is being computed

    returnLabel:
      If True, returns the label of the MetricSpec that was generated


<EOL>installPath = os.path.dirname(__file__)<EOL>outputFile = open(outputFilePath, "<STR_LIT:w>")<EOL>outputLines = []<EOL>inputLines = []<EOL>firstFile = True<EOL>for templateFileName in templateFileNames:<EOL><INDENT>if not firstFile:<EOL><INDENT>inputLines.extend([os.linesep]*<NUM_LIT:2>)<EOL><DEDENT>firstFile = False<EOL>inputFilePath = os.path.join(installPath, templateFileName)<EOL>inputFile = open(inputFilePath)<EOL>inputLines.extend(inputFile.readlines())<EOL>inputFile.close()<EOL><DEDENT>print("<STR_LIT>", len(inputLines), "<STR_LIT>")<EOL>for line in inputLines:<EOL><INDENT>tempLine = line<EOL>for k, v in replacementDict.items():<EOL><INDENT>if v is None:<EOL><INDENT>v = "<STR_LIT:None>"<EOL><DEDENT>tempLine = re.sub(k, v, tempLine)<EOL><DEDENT>outputFile.write(tempLine)<EOL><DEDENT>outputFile.close()<EOL>

Generates a file by applying token replacements to the given template
    file

    templateFileName:
                    A list of template file names; these files are assumed to be in
                    the same directory as the running experiment_generator.py script.
                    ExpGenerator will perform the substitution and concanetate
                    the files in the order they are specified

    outputFilePath: Absolute path of the output file

    replacementDict:
                    A dictionary of token/replacement pairs


width = <NUM_LIT:7><EOL>fieldName = fieldInfo['<STR_LIT>']<EOL>fieldType = fieldInfo['<STR_LIT>']<EOL>encoderChoicesList = []<EOL>if fieldType in ['<STR_LIT:float>', '<STR_LIT:int>']:<EOL><INDENT>aggFunction = '<STR_LIT>'<EOL>encoders = [None]<EOL>for n in (<NUM_LIT>, <NUM_LIT:50>, <NUM_LIT>, <NUM_LIT>):<EOL><INDENT>encoder = dict(type='<STR_LIT>', name=fieldName, fieldname=fieldName,<EOL>n=n, w=width, clipInput=True,space="<STR_LIT>")<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoder['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoder['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoders.append(encoder)<EOL><DEDENT>encoderChoicesList.append(encoders)<EOL><DEDENT>elif fieldType == '<STR_LIT:string>':<EOL><INDENT>aggFunction = '<STR_LIT>'<EOL>encoders = [None]<EOL>encoder = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>, w=width)<EOL>encoders.append(encoder)<EOL>encoderChoicesList.append(encoders)<EOL><DEDENT>elif fieldType == '<STR_LIT>':<EOL><INDENT>aggFunction = '<STR_LIT>'<EOL>encoders = [None]<EOL>for radius in (<NUM_LIT:1>, <NUM_LIT:8>):<EOL><INDENT>encoder = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, timeOfDay=(width, radius))<EOL>encoders.append(encoder)<EOL><DEDENT>encoderChoicesList.append(encoders)<EOL>encoders = [None]<EOL>for radius in (<NUM_LIT:1>, <NUM_LIT:3>):<EOL><INDENT>encoder = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, dayOfWeek=(width, radius))<EOL>encoders.append(encoder)<EOL><DEDENT>encoderChoicesList.append(encoders)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (fieldType))<EOL><DEDENT>return (encoderChoicesList, aggFunction)<EOL>

Return a list of possible encoder parameter combinations for the given
    field and the default aggregation function to use. Each parameter combination
    is a dict defining the parameters for the encoder. Here is an example
    return value for the encoderChoicesList:

     [
       None,
       {'fieldname':'timestamp',
        'name': 'timestamp_timeOfDay',
        'type':'DateEncoder'
        'dayOfWeek': (7,1)
        },
       {'fieldname':'timestamp',
        'name': 'timestamp_timeOfDay',
        'type':'DateEncoder'
        'dayOfWeek': (7,3)
        },
    ],

    Parameters:
    --------------------------------------------------
    fieldInfo:      item from the 'includedFields' section of the
                      description JSON object

    retval:  (encoderChoicesList, aggFunction)
               encoderChoicesList: a list of encoder choice lists for this field.
                 Most fields will generate just 1 encoder choice list.
                 DateTime fields can generate 2 or more encoder choice lists,
                   one for dayOfWeek, one for timeOfDay, etc.
               aggFunction: name of aggregation function to use for this
                             field type


<EOL>encoderChoicesList = []<EOL>for fieldInfo in includedFields:<EOL><INDENT>fieldName = fieldInfo['<STR_LIT>']<EOL>(choicesList, aggFunction) = _generateEncoderChoicesV1(fieldInfo)<EOL>encoderChoicesList.extend(choicesList)<EOL><DEDENT>encoderSpecsList = []<EOL>for encoderChoices in encoderChoicesList:<EOL><INDENT>encoder = encoderChoices[-<NUM_LIT:1>]<EOL>for c in _ILLEGAL_FIELDNAME_CHARACTERS:<EOL><INDENT>if encoder['<STR_LIT:name>'].find(c) >= <NUM_LIT:0>:<EOL><INDENT>raise _ExpGeneratorException("<STR_LIT>" % (<EOL>c, encoder['<STR_LIT:name>']))<EOL><DEDENT><DEDENT>encoderSpecsList.append("<STR_LIT>" % (<EOL>_quoteAndEscape(encoder['<STR_LIT:name>']),<EOL><NUM_LIT:2>*_ONE_INDENT,<EOL>pprint.pformat(encoder, indent=<NUM_LIT:2>*_INDENT_STEP)))<EOL><DEDENT>encoderSpecsStr = '<STR_LIT>'.join(encoderSpecsList)<EOL>permEncoderChoicesList = []<EOL>for encoderChoices in encoderChoicesList:<EOL><INDENT>permEncoderChoicesList.append("<STR_LIT>" % (<EOL>_quoteAndEscape(encoderChoices[-<NUM_LIT:1>]['<STR_LIT:name>']),<EOL>pprint.pformat(encoderChoices, indent=<NUM_LIT:2>*_INDENT_STEP)))<EOL><DEDENT>permEncoderChoicesStr = '<STR_LIT:\n>'.join(permEncoderChoicesList)<EOL>permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, <NUM_LIT:1>,<EOL>indentFirstLine=False)<EOL>return (encoderSpecsStr, permEncoderChoicesStr)<EOL>

Generate and return the following encoder related substitution variables:

    encoderSpecsStr:
      For the base description file, this string defines the default
      encoding dicts for each encoder. For example:
           '__gym_encoder' : {   'fieldname': 'gym',
            'n': 13,
            'name': 'gym',
            'type': 'SDRCategoryEncoder',
            'w': 7},
          '__address_encoder' : {   'fieldname': 'address',
            'n': 13,
            'name': 'address',
            'type': 'SDRCategoryEncoder',
            'w': 7}

    encoderSchemaStr:
      For the base description file, this is a list containing a
      DeferredDictLookup entry for each encoder. For example:
          [DeferredDictLookup('__gym_encoder'),
           DeferredDictLookup('__address_encoder'),
           DeferredDictLookup('__timestamp_timeOfDay_encoder'),
           DeferredDictLookup('__timestamp_dayOfWeek_encoder'),
           DeferredDictLookup('__consumption_encoder')],

    permEncoderChoicesStr:
      For the permutations file, this defines the possible
      encoder dicts for each encoder. For example:
          '__timestamp_dayOfWeek_encoder': [
                       None,
                       {'fieldname':'timestamp',
                        'name': 'timestamp_timeOfDay',
                        'type':'DateEncoder'
                        'dayOfWeek': (7,1)
                        },
                       {'fieldname':'timestamp',
                        'name': 'timestamp_timeOfDay',
                        'type':'DateEncoder'
                        'dayOfWeek': (7,3)
                        },
                    ],

          '__field_consumption_encoder': [
                      None,
                      {'fieldname':'consumption',
                       'name': 'consumption',
                       'type':'AdaptiveScalarEncoder',
                       'n': 13,
                       'w': 7,
                        }
                     ]

    Parameters:
    --------------------------------------------------
    includedFields:  item from the 'includedFields' section of the
                      description JSON object. This is a list of dicts, each
                      dict defining the field name, type, and optional min
                      and max values.

    retval:  (encoderSpecsStr, encoderSchemaStr permEncoderChoicesStr)


permStr = "<STR_LIT>"<EOL>if encoderDict.get('<STR_LIT>', False):<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if key == '<STR_LIT:n>' and encoderDict['<STR_LIT:type>'] != '<STR_LIT>':<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:w>"] + <NUM_LIT:7>,<EOL>encoderDict["<STR_LIT:w>"] + <NUM_LIT>)<EOL><DEDENT>else:<EOL><INDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>else:<EOL><INDENT>if encoderDict["<STR_LIT:type>"] in ["<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:type>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if key == "<STR_LIT:n>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:w>"] + <NUM_LIT:1>,<EOL>encoderDict["<STR_LIT:w>"] + <NUM_LIT>)<EOL><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>if value and not "<STR_LIT>" in encoderDict:<EOL><INDENT>permStr += "<STR_LIT>"% (_quoteAndEscape("<STR_LIT>"), _quoteAndEscape("<STR_LIT>"))<EOL><DEDENT>encoderDict.pop("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>elif encoderDict["<STR_LIT:type>"] in ["<STR_LIT>"]:<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:type>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>elif encoderDict["<STR_LIT:type>"] in ["<STR_LIT>"]:<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:type>":<EOL><INDENT>continue<EOL><DEDENT>elif key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if key == "<STR_LIT>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:type>"])<EOL>permStr += "<STR_LIT>"<EOL>permStr += "<STR_LIT>" % (value[<NUM_LIT:0>])<EOL><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:type>"])<EOL>permStr += "<STR_LIT>"<EOL>permStr += "<STR_LIT>" % (value[<NUM_LIT:0>])<EOL><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:type>"])<EOL>permStr += "<STR_LIT>"<EOL>permStr += "<STR_LIT>" % (value)<EOL><DEDENT>else:<EOL><INDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(encoderDict["<STR_LIT:type>"]))<EOL><DEDENT><DEDENT>return permStr<EOL>

Generate the string that defines the permutations to apply for a given
    encoder.

    Parameters:
    -----------------------------------------------------------------------
    options: experiment params
    encoderDict: the encoder dict, which gets placed into the description.py

    For example, if the encoderDict contains:
        'consumption':     {
                  'clipInput': True,
                  'fieldname': u'consumption',
                  'n': 100,
                  'name': u'consumption',
                  'type': 'AdaptiveScalarEncoder',
                  'w': 21},

    The return string will contain:
      "PermuteEncoder(fieldName='consumption',
                      encoderClass='AdaptiveScalarEncoder',
                      w=21,
                      n=PermuteInt(28, 521),
                      clipInput=True)"


width = <NUM_LIT><EOL>encoderDictsList = []<EOL>if options['<STR_LIT>'] in ["<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"]:<EOL><INDENT>classifierOnlyField = options['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>classifierOnlyField = None<EOL><DEDENT>for fieldInfo in includedFields:<EOL><INDENT>fieldName = fieldInfo['<STR_LIT>']<EOL>fieldType = fieldInfo['<STR_LIT>']<EOL>if fieldType in ['<STR_LIT:float>', '<STR_LIT:int>']:<EOL><INDENT>runDelta = fieldInfo.get("<STR_LIT>", False)<EOL>if runDelta or "<STR_LIT>" in fieldInfo:<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>, w=width, clipInput=True)<EOL>if runDelta:<EOL><INDENT>encoderDict["<STR_LIT>"] = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>, w=width, clipInput=True)<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if ('<STR_LIT>' in fieldInfo and '<STR_LIT>' in fieldInfo)and (encoderDict['<STR_LIT:type>'] == '<STR_LIT>'):<EOL><INDENT>encoderDict['<STR_LIT:type>'] = '<STR_LIT>'<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL><DEDENT>elif fieldType == '<STR_LIT:string>':<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>+width, w=width)<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL><DEDENT>elif fieldType == '<STR_LIT>':<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, timeOfDay=(width, <NUM_LIT:1>))<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL>encoderDict = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, dayOfWeek=(width, <NUM_LIT:1>))<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL>encoderDict = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, weekend=(width))<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (fieldType))<EOL><DEDENT>if fieldName == classifierOnlyField:<EOL><INDENT>clEncoderDict = dict(encoderDict)<EOL>clEncoderDict['<STR_LIT>'] = True<EOL>clEncoderDict['<STR_LIT:name>'] = '<STR_LIT>'<EOL>encoderDictsList.append(clEncoderDict)<EOL>if options["<STR_LIT>"]["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>encoderDictsList.remove(encoderDict)<EOL><DEDENT><DEDENT><DEDENT>if options.get('<STR_LIT>') is not None:<EOL><INDENT>tempList=[]<EOL>for encoderDict in encoderDictsList:<EOL><INDENT>if encoderDict['<STR_LIT:name>'] in options['<STR_LIT>']:<EOL><INDENT>tempList.append(encoderDict)<EOL><DEDENT><DEDENT>encoderDictsList = tempList<EOL><DEDENT>encoderSpecsList = []<EOL>permEncoderChoicesList = []<EOL>for encoderDict in encoderDictsList:<EOL><INDENT>if encoderDict['<STR_LIT:name>'].find('<STR_LIT:\\>') >= <NUM_LIT:0>:<EOL><INDENT>raise _ExpGeneratorException("<STR_LIT>")<EOL><DEDENT>for c in _ILLEGAL_FIELDNAME_CHARACTERS:<EOL><INDENT>if encoderDict['<STR_LIT:name>'].find(c) >= <NUM_LIT:0>:<EOL><INDENT>raise _ExpGeneratorException("<STR_LIT>"  %(c, encoderDict['<STR_LIT:name>']))<EOL><DEDENT><DEDENT>constructorStr = _generatePermEncoderStr(options, encoderDict)<EOL>encoderKey = _quoteAndEscape(encoderDict['<STR_LIT:name>'])<EOL>encoderSpecsList.append("<STR_LIT>" % (<EOL>encoderKey,<EOL><NUM_LIT:2>*_ONE_INDENT,<EOL>pprint.pformat(encoderDict, indent=<NUM_LIT:2>*_INDENT_STEP)))<EOL>permEncoderChoicesList.append("<STR_LIT>" % (encoderKey, constructorStr))<EOL><DEDENT>encoderSpecsStr = '<STR_LIT>'.join(encoderSpecsList)<EOL>permEncoderChoicesStr = '<STR_LIT:\n>'.join(permEncoderChoicesList)<EOL>permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, <NUM_LIT:1>,<EOL>indentFirstLine=True)<EOL>return (encoderSpecsStr, permEncoderChoicesStr)<EOL>

Generate and return the following encoder related substitution variables:

    encoderSpecsStr:
      For the base description file, this string defines the default
      encoding dicts for each encoder. For example:

           __gym_encoder = {   'fieldname': 'gym',
            'n': 13,
            'name': 'gym',
            'type': 'SDRCategoryEncoder',
            'w': 7},
          __address_encoder = {   'fieldname': 'address',
            'n': 13,
            'name': 'address',
            'type': 'SDRCategoryEncoder',
            'w': 7}

    permEncoderChoicesStr:
      For the permutations file, this defines the possible
      encoder dicts for each encoder. For example:

          '__gym_encoder' : PermuteEncoder('gym', 'SDRCategoryEncoder', w=7,
              n=100),

          '__address_encoder' : PermuteEncoder('address', 'SDRCategoryEncoder',
                w=7, n=100),

          '__timestamp_dayOfWeek_encoder' : PermuteEncoder('timestamp',
              'DateEncoder.timeOfDay', w=7, radius=PermuteChoices([1, 8])),

          '__consumption_encoder': PermuteEncoder('consumption', 'AdaptiveScalarEncoder',
              w=7, n=PermuteInt(13, 500, 20), minval=0,
              maxval=PermuteInt(100, 300, 25)),

    Parameters:
    --------------------------------------------------
    includedFields:  item from the 'includedFields' section of the
                      description JSON object. This is a list of dicts, each
                      dict defining the field name, type, and optional min
                      and max values.

    retval:  (encoderSpecsStr permEncoderChoicesStr)


<EOL>if '<STR_LIT>' not in options:<EOL><INDENT>prediction = options.get('<STR_LIT>', {InferenceType.TemporalNextStep:<EOL>{'<STR_LIT>':True}})<EOL>inferenceType = None<EOL>for infType, value in prediction.items():<EOL><INDENT>if value['<STR_LIT>']:<EOL><INDENT>inferenceType = infType<EOL>break<EOL><DEDENT><DEDENT>if inferenceType == '<STR_LIT>':<EOL><INDENT>inferenceType = InferenceType.TemporalNextStep<EOL><DEDENT>if inferenceType != InferenceType.TemporalNextStep:<EOL><INDENT>raise _ExpGeneratorException("<STR_LIT>"  %(inferenceType))<EOL><DEDENT>options['<STR_LIT>'] = inferenceType<EOL><DEDENT>if '<STR_LIT>' in options:<EOL><INDENT>if '<STR_LIT>' not in options:<EOL><INDENT>options['<STR_LIT>'] = {'<STR_LIT>': options['<STR_LIT>']}<EOL><DEDENT>elif '<STR_LIT>' not in options['<STR_LIT>']:<EOL><INDENT>options['<STR_LIT>']['<STR_LIT>'] = options['<STR_LIT>']<EOL><DEDENT><DEDENT>


def _getPropertyValue(schema, propertyName, options):

if propertyName not in options:<EOL><INDENT>paramsSchema = schema['<STR_LIT>'][propertyName]<EOL>if '<STR_LIT:default>' in paramsSchema:<EOL><INDENT>options[propertyName] = paramsSchema['<STR_LIT:default>']<EOL><DEDENT>else:<EOL><INDENT>options[propertyName] = None<EOL><DEDENT><DEDENT>

Checks to see if property is specified in 'options'. If not, reads the
    default value from the schema


installPath = os.path.dirname(os.path.abspath(__file__))<EOL>schemaFilePath = os.path.join(installPath, "<STR_LIT>")<EOL>return json.loads(open(schemaFilePath, '<STR_LIT:r>').read())<EOL>

Returns the experiment description schema. This implementation loads it in
from file experimentDescriptionSchema.json.

Parameters:
--------------------------------------------------------------------------
Returns:    returns a dict representing the experiment description schema.