<EOL>if inferenceElement in (InferenceElement.prediction,<EOL>InferenceElement.encodings):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if inferenceElement in (InferenceElement.anomalyScore,<EOL>InferenceElement.anomalyLabel,<EOL>InferenceElement.classification,<EOL>InferenceElement.classConfidences):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>if inferenceElement in (InferenceElement.multiStepPredictions,<EOL>InferenceElement.multiStepBestPredictions):<EOL><INDENT>return int(key)<EOL><DEDENT>return <NUM_LIT:0><EOL>

Returns the number of records that elapse between when an inference is
        made and when the corresponding input record will appear. For example, a
        multistep prediction for 3 timesteps out will have a delay of 3

        Parameters:
        -----------------------------------------------------------------------

        inferenceElement:   The InferenceElement value being delayed
        key:                If the inference is a dictionary type, this specifies
                            key for the sub-inference that is being delayed


if InferenceType.__temporalInferenceTypes is None:<EOL><INDENT>InferenceType.__temporalInferenceTypes =set([InferenceType.TemporalNextStep,<EOL>InferenceType.TemporalClassification,<EOL>InferenceType.TemporalAnomaly,<EOL>InferenceType.TemporalMultiStep,<EOL>InferenceType.NontemporalMultiStep])<EOL><DEDENT>return inferenceType in InferenceType.__temporalInferenceTypes<EOL>

Returns True if the inference type is 'temporal', i.e. requires a
        temporal memory in the network.


return InferenceElement.__inferenceInputMap.get(inferenceElement, None)<EOL>

Get the sensor input element that corresponds to the given inference
        element. This is mainly used for metrics and prediction logging


def __init__(self,  jobID, jobsDAO, logLevel = None):

self._jobID = jobID<EOL>self._cjDB = jobsDAO<EOL>self._lastUpdateAttemptTime = <NUM_LIT:0><EOL>initLogging(verbose = True)<EOL>self.logger = logging.getLogger("<STR_LIT:.>".join( ['<STR_LIT>',<EOL>self.__class__.__module__, self.__class__.__name__]))<EOL>if logLevel is not None:<EOL><INDENT>self.logger.setLevel(logLevel)<EOL><DEDENT>self.logger.info("<STR_LIT>" % str(jobID))<EOL>


updateInterval = time.time() - self._lastUpdateAttemptTime<EOL>if updateInterval < self._MIN_UPDATE_INTERVAL and not forceUpdate:<EOL><INDENT>return<EOL><DEDENT>self.logger.info("<STR_LIT>""<STR_LIT>"%(self._jobID,<EOL>time.time(),<EOL>self._lastUpdateAttemptTime))<EOL>timestampUpdated = self._cjDB.jobUpdateSelectionSweep(self._jobID,<EOL>self._MIN_UPDATE_INTERVAL)<EOL>if not timestampUpdated:<EOL><INDENT>self.logger.info("<STR_LIT>""<STR_LIT>"%(self._jobID, self._lastUpdateAttemptTime))<EOL>if not forceUpdate:<EOL><INDENT>return<EOL><DEDENT><DEDENT>self._lastUpdateAttemptTime = time.time()<EOL>self.logger.info("<STR_LIT>"%(self._jobID, self._lastUpdateAttemptTime))<EOL>minUpdateRecords = self._MIN_UPDATE_THRESHOLD<EOL>jobResults = self._getJobResults()<EOL>if forceUpdate or jobResults is None:<EOL><INDENT>minUpdateRecords = <NUM_LIT:0><EOL><DEDENT>candidateIDs, bestMetric = self._cjDB.modelsGetCandidates(self._jobID, minUpdateRecords)<EOL>self.logger.info("<STR_LIT>"%(candidateIDs, bestMetric, self._jobID))<EOL>if len(candidateIDs) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>self._jobUpdateCandidate(candidateIDs[<NUM_LIT:0>], bestMetric, results=jobResults)<EOL>

Chooses the best model for a given job.

        Parameters
        -----------------------------------------------------------------------
        forceUpdate:  (True/False). If True, the update will ignore all the
                      restrictions on the minimum time to update and the minimum
                      number of records to update. This should typically only be
                      set to true if the model has completed running


matchingReportKeys = []<EOL>for keyRE in reportKeyREs:<EOL><INDENT>matchObj = re.compile(keyRE)<EOL>found = False<EOL>for keyName in allReportKeys:<EOL><INDENT>match = matchObj.match(keyName)<EOL>if match and match.end() == len(keyName):<EOL><INDENT>matchingReportKeys.append(keyName)<EOL>found = True<EOL><DEDENT><DEDENT>if not found:<EOL><INDENT>raise _BadKeyError(keyRE)<EOL><DEDENT><DEDENT>return matchingReportKeys<EOL>

Extract all items from the 'allKeys' list whose key matches one of the regular
expressions passed in 'reportKeys'.

Parameters:
----------------------------------------------------------------------------
reportKeyREs:     List of regular expressions
allReportKeys:    List of all keys

retval:         list of keys from allReportKeys that match the regular expressions
                  in 'reportKeyREs'
                If an invalid regular expression was included in 'reportKeys',
                  then BadKeyError() is raised


subKeys = itemName.split('<STR_LIT::>')<EOL>subResults = results<EOL>for subKey in subKeys:<EOL><INDENT>subResults = subResults[subKey]<EOL><DEDENT>return subResults<EOL>

Get a specific item by name out of the results dict.

The format of itemName is a string of dictionary keys separated by colons,
each key being one level deeper into the results dict. For example,
'key1:key2' would fetch results['key1']['key2'].

If itemName is not found in results, then None is returned


with open(inputFilePath) as fileObj:<EOL><INDENT>value = json.load(fileObj)<EOL><DEDENT>return value<EOL>

Loads a json value from a file and converts it to the corresponding python
    object.

    inputFilePath:
                    Path of the json file;

    Returns:
                    python value that represents the loaded json value


assert type(string) in (str,)<EOL>return pprint.pformat(string)<EOL>

string:   input string (ascii or unicode)

Returns:  a quoted string with characters that are represented in python via
          escape sequences converted to those escape sequences


return "<STR_LIT>" + str(uuid.uuid1())<EOL>


def filterResults(allResults, reportKeys, optimizeKey=None):

<EOL>optimizeDict = dict()<EOL>allReportKeys = set()<EOL>_appendReportKeys(keys=allReportKeys, prefix='<STR_LIT>', results=allResults)<EOL>matchingKeys = _matchReportKeys(reportKeys, allReportKeys)<EOL>reportDict = dict()<EOL>for keyName in matchingKeys:<EOL><INDENT>value = _getReportItem(keyName, allResults)<EOL>reportDict[keyName] = value<EOL><DEDENT>if optimizeKey is not None:<EOL><INDENT>matchingKeys = _matchReportKeys([optimizeKey], allReportKeys)<EOL>if len(matchingKeys) == <NUM_LIT:0>:<EOL><INDENT>raise _BadKeyError(optimizeKey)<EOL><DEDENT>elif len(matchingKeys) > <NUM_LIT:1>:<EOL><INDENT>raise _BadOptimizeKeyError(optimizeKey, matchingKeys)<EOL><DEDENT>optimizeKeyFullName = matchingKeys[<NUM_LIT:0>]<EOL>value = _getReportItem(optimizeKeyFullName, allResults)<EOL>optimizeDict[optimizeKeyFullName] = value<EOL>reportDict[optimizeKeyFullName] = value<EOL><DEDENT>return(reportDict, optimizeDict)<EOL>

Given the complete set of results generated by an experiment (passed in
    'results'), filter out and return only the ones the caller wants, as
    specified through 'reportKeys' and 'optimizeKey'.

    A report key is a string of key names separated by colons, each key being one
    level deeper into the experiment results dict. For example, 'key1:key2'.

    Parameters:
    -------------------------------------------------------------------------
    results:             dict of all results generated by an experiment
    reportKeys:          list of items from the results dict to include in
                         the report. These can be regular expressions.
    optimizeKey:         Which report item, if any, we will be optimizing for. This can
                         also be a regular expression, but is an error if it matches
                         more than one key from the experiment's results.
    retval:  (reportDict, optimizeDict)
                reportDict: a dictionary of the metrics named by desiredReportKeys
                optimizeDict: A dictionary containing 1 item: the full name and
                      value of the metric identified by the optimizeKey


str = getCopyrightHead() +"<EOL>his file defines parameters for a prediction experiment.<EOL><INDENT>IMPORTANT!!!<EOL><DEDENT>is params file is dynamically generated by the RunExperimentPermutations<EOL>ript. Any changes made manually will be over-written the next time<EOL>nExperimentPermutations is run!!!<EOL><INDENT>nupic.frameworks.opf.exp_description_helpers import importBaseDescription<EOL><DEDENT>e sub-experiment configuration<EOL>ig ={<EOL>return str<EOL>

This is the first portion of every sub-experiment params file we generate. Between
the head and the tail are the experiment specific options.


<EOL>for act in self.__activities:<EOL><INDENT>if not act.iteratorHolder[<NUM_LIT:0>]:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>next(act.iteratorHolder[<NUM_LIT:0>])<EOL><DEDENT>except StopIteration:<EOL><INDENT>act.cb()<EOL>if act.repeating:<EOL><INDENT>act.iteratorHolder[<NUM_LIT:0>] = iter(range(act.period))<EOL><DEDENT>else:<EOL><INDENT>act.iteratorHolder[<NUM_LIT:0>] = None<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Activity tick handler; services all activities

        Returns:      True if controlling iterator says it's okay to keep going;
                      False to stop


super(_HyperSearchJob, self).__init__(nupicJobID)<EOL>self.__expectedNumModels = None<EOL>


return self.__unwrapResults().optimizationMetrics<EOL>

Retrives a dictionary of metrics designagted for optimization
        Parameters:
        ----------------------------------------------------------------------
        retval:         a dictionary of optimization metrics that were collected
                        for the model or an empty dictionary if there aren't any.


self.__searchJob = hyperSearchJob<EOL>self.__searchJobID = hyperSearchJob.getJobID()<EOL>self.__sortedMetricsKeys = sorted(metricsKeys)<EOL>self.__outputDirAbsPath = os.path.abspath(outputDirAbsPath)<EOL>self.__outputLabel = outputLabel<EOL>self.__replaceReport = replaceReport<EOL>self.__sortedVariableNames=searchVar<EOL>self.__csvFileObj = None<EOL>self.__reportCSVPath = None<EOL>self.__backupCSVPath = None<EOL>

Parameters:
----------------------------------------------------------------------
hyperSearchJob: _HyperSearchJob instance
metricsKeys:    sequence of report metrics key names to include in report
outputDirAbsPath:
                Directory for creating report CSV file (absolute path)
outputLabel:    A string label to incorporate into report CSV file name
replaceReport:  True to replace existing report csv, if any; False to
                append to existing report csv, if any
retval:         nothing


process = subprocess.Popen(["<STR_LIT>", "<STR_LIT>"], stdout=subprocess.PIPE)<EOL>stdout = process.communicate()[<NUM_LIT:0>]<EOL>result = process.returncode<EOL>if result != <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>running = False<EOL>for line in stdout.split("<STR_LIT:\n>"):<EOL><INDENT>if "<STR_LIT>" in line and "<STR_LIT>" in line:<EOL><INDENT>running = True<EOL>break<EOL><DEDENT><DEDENT>return running<EOL>

Return true if the engine services are running


assert self.isFinished(), "<STR_LIT>" % self<EOL>return "<STR_LIT:%s>" % self.__rawInfo.endTime<EOL>

Returns mode evaluation end time.

        NOTE: it's an error to call this method if isFinished() would return False.

        Parameters:
        ----------------------------------------------------------------------
        retval:         model evaluation end time


@classmethod<EOL><INDENT>def loadSavedHyperSearchJob(cls, permWorkDir, outputLabel):<DEDENT>

jobID = cls.__loadHyperSearchJobID(permWorkDir=permWorkDir,<EOL>outputLabel=outputLabel)<EOL>searchJob = _HyperSearchJob(nupicJobID=jobID)<EOL>return searchJob<EOL>

Instantiates a _HyperSearchJob instance from info saved in file

        Parameters:
        ----------------------------------------------------------------------
        permWorkDir: Directory path for saved jobID file
        outputLabel: Label string for incorporating into file name for saved jobID
        retval:      _HyperSearchJob instance; raises exception if not found


if self.__csvFileObj is not None:<EOL><INDENT>self.__csvFileObj.close()<EOL>self.__csvFileObj = None<EOL>print("<STR_LIT>" % (self.__reportCSVPath,))<EOL>if self.__backupCSVPath:<EOL><INDENT>print("<STR_LIT>" %(self.__backupCSVPath,))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>


<EOL>if hyperSearchJob is None:<EOL><INDENT>hyperSearchJob = cls.loadSavedHyperSearchJob(<EOL>permWorkDir=options["<STR_LIT>"],<EOL>outputLabel=options["<STR_LIT>"])<EOL><DEDENT>modelIDs = hyperSearchJob.queryModelIDs()<EOL>bestModel = None<EOL>metricstmp = set()<EOL>searchVar = set()<EOL>for modelInfo in _iterModels(modelIDs):<EOL><INDENT>if modelInfo.isFinished():<EOL><INDENT>vars = list(modelInfo.getParamLabels().keys())<EOL>searchVar.update(vars)<EOL>metrics = modelInfo.getReportMetrics()<EOL>metricstmp.update(list(metrics.keys()))<EOL><DEDENT><DEDENT>if metricsKeys is None:<EOL><INDENT>metricsKeys = metricstmp<EOL><DEDENT>reportWriter = _ReportCSVWriter(hyperSearchJob=hyperSearchJob,<EOL>metricsKeys=metricsKeys,<EOL>searchVar=searchVar,<EOL>outputDirAbsPath=options["<STR_LIT>"],<EOL>outputLabel=options["<STR_LIT>"],<EOL>replaceReport=replaceReport)<EOL>modelStats = _ModelStats()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>searchParams = hyperSearchJob.getParams()<EOL>(optimizationMetricKey, maximizeMetric) = (<EOL>_PermutationUtils.getOptimizationMetricInfo(searchParams))<EOL>formatStr = None<EOL>foundMetricsKeySet = set(metricsKeys)<EOL>sortedMetricsKeys = []<EOL>jobInfo = _clientJobsDB().jobInfo(hyperSearchJob.getJobID())<EOL>if jobInfo.cancel == <NUM_LIT:1>:<EOL><INDENT>raise Exception(jobInfo.workerCompletionMsg)<EOL><DEDENT>try:<EOL><INDENT>results = json.loads(jobInfo.results)<EOL><DEDENT>except Exception as e:<EOL><INDENT>print("<STR_LIT>""<STR_LIT>")<EOL>print("<STR_LIT>", jobInfo)<EOL>print("<STR_LIT>", jobInfo.results)<EOL>print("<STR_LIT>", e)<EOL>raise<EOL><DEDENT>bestModelNum = results["<STR_LIT>"]<EOL>bestModelIterIndex = None<EOL>totalWallTime = <NUM_LIT:0><EOL>totalRecords = <NUM_LIT:0><EOL>scoreModelIDDescList = []<EOL>for (i, modelInfo) in enumerate(_iterModels(modelIDs)):<EOL><INDENT>reportWriter.emit(modelInfo)<EOL>totalRecords+=modelInfo.getNumRecords()<EOL>format = "<STR_LIT>"<EOL>startTime = modelInfo.getStartTime()<EOL>if modelInfo.isFinished():<EOL><INDENT>endTime = modelInfo.getEndTime()<EOL>st = datetime.strptime(startTime, format)<EOL>et = datetime.strptime(endTime, format)<EOL>totalWallTime+=(et-st).seconds<EOL><DEDENT>modelStats.update(modelInfo)<EOL>expDesc = modelInfo.getModelDescription()<EOL>reportMetrics = modelInfo.getReportMetrics()<EOL>optimizationMetrics = modelInfo.getOptimizationMetrics()<EOL>if modelInfo.getModelID() == bestModelNum:<EOL><INDENT>bestModel = modelInfo<EOL>bestModelIterIndex=i<EOL>bestMetric = list(optimizationMetrics.values())[<NUM_LIT:0>]<EOL><DEDENT>if optimizationMetrics:<EOL><INDENT>assert len(optimizationMetrics) == <NUM_LIT:1>, (<EOL>"<STR_LIT>" % (<EOL>len(optimizationMetrics), optimizationMetrics, modelInfo))<EOL><DEDENT>if modelInfo.getCompletionReason().isEOF():<EOL><INDENT>scoreModelIDDescList.append((list(optimizationMetrics.values())[<NUM_LIT:0>],<EOL>modelInfo.getModelID(),<EOL>modelInfo.getGeneratedDescriptionFile(),<EOL>modelInfo.getParamLabels()))<EOL><DEDENT>print("<STR_LIT>" % (i, modelInfo, expDesc))<EOL>if (modelInfo.isFinished() and<EOL>not (modelInfo.getCompletionReason().isStopped or<EOL>modelInfo.getCompletionReason().isEOF())):<EOL><INDENT>print("<STR_LIT>" % modelInfo.getCompletionMsg())<EOL><DEDENT>if reportMetrics:<EOL><INDENT>foundMetricsKeySet.update(iter(reportMetrics.keys()))<EOL>if len(sortedMetricsKeys) != len(foundMetricsKeySet):<EOL><INDENT>sortedMetricsKeys = sorted(foundMetricsKeySet)<EOL>maxKeyLen = max([len(k) for k in sortedMetricsKeys])<EOL>formatStr = "<STR_LIT>" % (maxKeyLen+<NUM_LIT:2>)<EOL><DEDENT>for key in sortedMetricsKeys:<EOL><INDENT>if key in reportMetrics:<EOL><INDENT>if key == optimizationMetricKey:<EOL><INDENT>m = "<STR_LIT>" % reportMetrics[key]<EOL><DEDENT>else:<EOL><INDENT>m = "<STR_LIT>" % reportMetrics[key]<EOL><DEDENT>print(formatStr % (key+"<STR_LIT::>"), m)<EOL><DEDENT><DEDENT>print()<EOL><DEDENT><DEDENT>print("<STR_LIT>")<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>len(modelIDs),<EOL>("<STR_LIT>"<EOL>if (modelStats.numCompletedKilled + modelStats.numCompletedEOF) ==<EOL>len(modelIDs)<EOL>else "<STR_LIT>" % (<EOL>len(modelIDs) - (<EOL>modelStats.numCompletedKilled + modelStats.numCompletedEOF +<EOL>modelStats.numCompletedStopped)))))<EOL>if modelStats.numStatusOther > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>modelStats.numStatusOther))<EOL><DEDENT>print("<STR_LIT>" % modelStats.numStatusWaitingToStart)<EOL>print("<STR_LIT>" % modelStats.numStatusRunning)<EOL>print("<STR_LIT>" % modelStats.numStatusCompleted)<EOL>if modelStats.numCompletedOther > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>modelStats.numCompletedOther))<EOL><DEDENT>print("<STR_LIT>" % modelStats.numCompletedEOF)<EOL>print("<STR_LIT>" % modelStats.numCompletedStopped)<EOL>print("<STR_LIT>" % modelStats.numCompletedOrphaned)<EOL>print("<STR_LIT>" % modelStats.numCompletedKilled)<EOL>print("<STR_LIT>" % modelStats.numCompletedError)<EOL>assert modelStats.numStatusOther == <NUM_LIT:0>, "<STR_LIT>" % (<EOL>modelStats.numStatusOther)<EOL>assert modelStats.numCompletedOther == <NUM_LIT:0>, "<STR_LIT>" % (<EOL>modelStats.numCompletedOther)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>print()<EOL>global gCurrentSearch<EOL>jobStatus = hyperSearchJob.getJobStatus(gCurrentSearch._workers)<EOL>jobResults = jobStatus.getResults()<EOL>if "<STR_LIT>" in jobResults:<EOL><INDENT>print("<STR_LIT>")<EOL>pprint.pprint(jobResults["<STR_LIT>"], indent=<NUM_LIT:4>)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>if bestModel is not None:<EOL><INDENT>maxKeyLen = max([len(k) for k in sortedMetricsKeys])<EOL>maxKeyLen = max(maxKeyLen, len(optimizationMetricKey))<EOL>formatStr = "<STR_LIT>" % (maxKeyLen+<NUM_LIT:2>)<EOL>bestMetricValue = list(bestModel.getOptimizationMetrics().values())[<NUM_LIT:0>]<EOL>optimizationMetricName = list(bestModel.getOptimizationMetrics().keys())[<NUM_LIT:0>]<EOL>print()<EOL>print("<STR_LIT>" % (<EOL>optimizationMetricName, maximizeMetric))<EOL>print("<STR_LIT>" % (<EOL>bestModelIterIndex, bestModel, bestModel.getModelDescription()))<EOL>print(formatStr % (optimizationMetricName+"<STR_LIT::>"), bestMetricValue)<EOL>print()<EOL>print("<STR_LIT>"  % totalRecords)<EOL>print()<EOL>print("<STR_LIT>" % totalWallTime)<EOL>hsJobParams = hyperSearchJob.getParams()<EOL><DEDENT>if options["<STR_LIT>"] > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>" % (<EOL>options["<STR_LIT>"]))<EOL>scoreModelIDDescList.sort()<EOL>scoreModelIDDescList = scoreModelIDDescList[<EOL><NUM_LIT:0>:options["<STR_LIT>"]]<EOL>i = -<NUM_LIT:1><EOL>for (score, modelID, description, paramLabels) in scoreModelIDDescList:<EOL><INDENT>i += <NUM_LIT:1><EOL>outDir = os.path.join(options["<STR_LIT>"], "<STR_LIT>" % (i))<EOL>print("<STR_LIT>" %(modelID, outDir))<EOL>if not os.path.exists(outDir):<EOL><INDENT>os.makedirs(outDir)<EOL><DEDENT>base_description_path = os.path.join(options["<STR_LIT>"],<EOL>"<STR_LIT>")<EOL>base_description_relpath = os.path.relpath(base_description_path,<EOL>start=outDir)<EOL>description = description.replace(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>" % base_description_relpath)<EOL>fd = open(os.path.join(outDir, "<STR_LIT>"), "<STR_LIT:wb>")<EOL>fd.write(description)<EOL>fd.close()<EOL>fd = open(os.path.join(outDir, "<STR_LIT>"), "<STR_LIT:wb>")<EOL>writer = csv.writer(fd)<EOL>colNames = list(paramLabels.keys())<EOL>colNames.sort()<EOL>writer.writerow(colNames)<EOL>row = [paramLabels[x] for x in colNames]<EOL>writer.writerow(row)<EOL>fd.close()<EOL>print("<STR_LIT>")<EOL>mod = imp.load_source("<STR_LIT:description>", os.path.join(outDir,<EOL>"<STR_LIT>"))<EOL>model_description = mod.descriptionInterface.getModelDescription()<EOL>fd = open(os.path.join(outDir, "<STR_LIT>"), "<STR_LIT:wb>")<EOL>fd.write("<STR_LIT>" % (getCopyrightHead(),<EOL>pprint.pformat(model_description)))<EOL>fd.close()<EOL><DEDENT>print()<EOL><DEDENT>reportWriter.finalize()<EOL>return model_description<EOL>

Prints all available results in the given HyperSearch job and emits
        model information to the permutations report csv.

        The job may be completed or still in progress.

        Parameters:
        ----------------------------------------------------------------------
        options:        NupicRunPermutations options dict
        replaceReport:  True to replace existing report csv, if any; False to
                        append to existing report csv, if any
        hyperSearchJob: _HyperSearchJob instance; if None, will get it from saved
                        jobID, if any
        metricsKeys:    sequence of report metrics key names to include in report;
                        if None, will pre-scan all modelInfos to generate a complete
                        list of metrics key names.
        retval:         model parameters


assert isinstance(s, str),"<STR_LIT>" % (type(str), type(s), s)<EOL>s = s.replace("<STR_LIT:\\>", "<STR_LIT>")<EOL>s = s.replace("<STR_LIT:\n>", "<STR_LIT>")<EOL>s = s.replace("<STR_LIT:\t>", "<STR_LIT>")<EOL>s = s.replace("<STR_LIT:U+002C>", "<STR_LIT:\t>")<EOL>return s<EOL>

Escape commas, tabs, newlines and dashes in a string

    Commas are encoded as tabs


if searchJobParams["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>search = HypersearchV2(searchParams=searchJobParams)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(searchJobParams["<STR_LIT>"]))<EOL><DEDENT>info = search.getOptimizationMetricInfo()<EOL>return info<EOL>

Retrives the optimization key name and optimization function.

        Parameters:
        ---------------------------------------------------------
        searchJobParams:
                        Parameter for passing as the searchParams arg to
                        Hypersearch constructor.
        retval:       (optimizationMetricKey, maximize)
                      optimizationMetricKey: which report key to optimize for
                      maximize: True if we should try and maximize the optimizeKey
                        metric. False if we should minimize it.


self.__cjDAO = _clientJobsDB()<EOL>self._options = options<EOL>self.__searchJob = None<EOL>self.__foundMetrcsKeySet = set()<EOL>self._workers = None<EOL>return<EOL>


if self.__cachedResults is None:<EOL><INDENT>if self.__rawInfo.results is not None:<EOL><INDENT>resultList = json.loads(self.__rawInfo.results)<EOL>assert len(resultList) == <NUM_LIT:2>,"<STR_LIT>" % (<EOL>len(resultList), resultList)<EOL>self.__cachedResults = self.ModelResults(<EOL>reportMetrics=resultList[<NUM_LIT:0>],<EOL>optimizationMetrics=resultList[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>self.__cachedResults = self.ModelResults(<EOL>reportMetrics={},<EOL>optimizationMetrics={})<EOL><DEDENT><DEDENT>return self.__cachedResults<EOL>

Unwraps self.__rawInfo.results and caches it in self.__cachedResults;
        Returns the unwrapped params

        Parameters:
        ----------------------------------------------------------------------
        retval:         ModelResults namedtuple instance


waiting = (self.__rawInfo.status == self.__nupicModelStatus_notStarted)<EOL>return waiting<EOL>


return self.__params<EOL>

Semi-private method for retrieving the job-specific params

        Parameters:
        ----------------------------------------------------------------------
        retval:         Job params dict corresponding to the JSON params value
                        returned by ClientJobsDAO.jobInfo()


assert self.__searchJob is not None<EOL>return self.__searchJob<EOL>

Retrieves the runner's _HyperSearchJob instance; NOTE: only available
        after run().

        Parameters:
        ----------------------------------------------------------------------
        retval:         _HyperSearchJob instance or None


<EOL>params = _ClientJobUtils.makeSearchJobParamsDict(options=self._options,<EOL>forRunning=True)<EOL>if self._options["<STR_LIT:action>"] == "<STR_LIT>":<EOL><INDENT>args = [sys.argv[<NUM_LIT:0>], "<STR_LIT>" % (json.dumps(params))]<EOL>print()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>jobID = hypersearch_worker.main(args)<EOL><DEDENT>else:<EOL><INDENT>cmdLine = _setUpExports(self._options["<STR_LIT>"])<EOL>cmdLine += "<STR_LIT>"<EOL>maxWorkers = self._options["<STR_LIT>"]<EOL>jobID = self.__cjDAO.jobInsert(<EOL>client="<STR_LIT>",<EOL>cmdLine=cmdLine,<EOL>params=json.dumps(params),<EOL>minimumWorkers=<NUM_LIT:1>,<EOL>maximumWorkers=maxWorkers,<EOL>jobType=self.__cjDAO.JOB_TYPE_HS)<EOL>cmdLine = "<STR_LIT>""<STR_LIT>" % (jobID)<EOL>self._launchWorkers(cmdLine, maxWorkers)<EOL><DEDENT>searchJob = _HyperSearchJob(jobID)<EOL>self.__saveHyperSearchJobID(<EOL>permWorkDir=self._options["<STR_LIT>"],<EOL>outputLabel=self._options["<STR_LIT>"],<EOL>hyperSearchJob=searchJob)<EOL>if self._options["<STR_LIT:action>"] == "<STR_LIT>":<EOL><INDENT>print("<STR_LIT>" % (jobID))<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>" % (jobID))<EOL>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (cmdLine,))<EOL><DEDENT>return searchJob<EOL>

Starts HyperSearch as a worker or runs it inline for the "dryRun" action

        Parameters:
        ----------------------------------------------------------------------
        retval:         the new _HyperSearchJob instance representing the
                        HyperSearch job


jobID = self.getJobID()<EOL>modelCounterPairs = _clientJobsDB().modelsGetUpdateCounters(jobID)<EOL>modelIDs = tuple(x[<NUM_LIT:0>] for x in modelCounterPairs)<EOL>return modelIDs<EOL>

Queuries DB for model IDs of all currently instantiated models
        associated with this HyperSearch job.

        See also: _iterModels()

        Parameters:
        ----------------------------------------------------------------------
        retval:         A sequence of Nupic modelIDs


assert self.__searchJob is not None<EOL>jobID = self.__searchJob.getJobID()<EOL>startTime = time.time()<EOL>lastUpdateTime = datetime.now()<EOL>expectedNumModels = self.__searchJob.getExpectedNumModels(<EOL>searchMethod = self._options["<STR_LIT>"])<EOL>lastNumFinished = <NUM_LIT:0><EOL>finishedModelIDs = set()<EOL>finishedModelStats = _ModelStats()<EOL>lastWorkerState = None<EOL>lastJobResults = None<EOL>lastModelMilestones = None<EOL>lastEngStatus = None<EOL>hyperSearchFinished = False<EOL>while not hyperSearchFinished:<EOL><INDENT>jobInfo = self.__searchJob.getJobStatus(self._workers)<EOL>hyperSearchFinished = jobInfo.isFinished()<EOL>modelIDs = self.__searchJob.queryModelIDs()<EOL>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (<EOL>len(modelIDs), len(finishedModelIDs)))<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>checkModelIDs = []<EOL>for modelID in modelIDs:<EOL><INDENT>if modelID not in finishedModelIDs:<EOL><INDENT>checkModelIDs.append(modelID)<EOL><DEDENT><DEDENT>del modelIDs<EOL>if checkModelIDs:<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (len(checkModelIDs)))<EOL>errorCompletionMsg = None<EOL>for (i, modelInfo) in enumerate(_iterModels(checkModelIDs)):<EOL><INDENT>_emit(Verbosity.DEBUG,<EOL>"<STR_LIT>" % (i, modelInfo))<EOL>if modelInfo.isFinished():<EOL><INDENT>finishedModelIDs.add(modelInfo.getModelID())<EOL>finishedModelStats.update(modelInfo)<EOL>if (modelInfo.getCompletionReason().isError() and<EOL>not errorCompletionMsg):<EOL><INDENT>errorCompletionMsg = modelInfo.getCompletionMsg()<EOL><DEDENT>metrics = modelInfo.getReportMetrics()<EOL>self.__foundMetrcsKeySet.update(list(metrics.keys()))<EOL><DEDENT><DEDENT><DEDENT>numFinished = len(finishedModelIDs)<EOL>if numFinished != lastNumFinished:<EOL><INDENT>lastNumFinished = numFinished<EOL>if expectedNumModels is None:<EOL><INDENT>expModelsStr = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>expModelsStr = "<STR_LIT>" % (expectedNumModels)<EOL><DEDENT>stats = finishedModelStats<EOL>print(("<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>jobID,<EOL>numFinished,<EOL>expModelsStr,<EOL>(stats.numCompletedEOF+stats.numCompletedStopped),<EOL>"<STR_LIT>" if stats.numCompletedEOF else "<STR_LIT>",<EOL>stats.numCompletedEOF,<EOL>"<STR_LIT>" if stats.numCompletedStopped else "<STR_LIT>",<EOL>stats.numCompletedStopped,<EOL>"<STR_LIT>" if stats.numCompletedKilled else "<STR_LIT>",<EOL>stats.numCompletedKilled,<EOL>"<STR_LIT>" if stats.numCompletedError else "<STR_LIT:error>",<EOL>stats.numCompletedError,<EOL>"<STR_LIT>" if stats.numCompletedError else "<STR_LIT>",<EOL>stats.numCompletedOrphaned,<EOL>"<STR_LIT>" if stats.numCompletedOther else "<STR_LIT>",<EOL>stats.numCompletedOther)))<EOL>if errorCompletionMsg:<EOL><INDENT>print("<STR_LIT>" % errorCompletionMsg)<EOL><DEDENT><DEDENT>workerState = jobInfo.getWorkerState()<EOL>if workerState != lastWorkerState:<EOL><INDENT>print("<STR_LIT>" % (pprint.pformat(workerState,<EOL>indent=<NUM_LIT:4>)))<EOL>lastWorkerState = workerState<EOL><DEDENT>jobResults = jobInfo.getResults()<EOL>if jobResults != lastJobResults:<EOL><INDENT>print("<STR_LIT>"% (pprint.pformat(jobResults, indent=<NUM_LIT:4>), time.time()-startTime))<EOL>lastJobResults = jobResults<EOL><DEDENT>modelMilestones = jobInfo.getModelMilestones()<EOL>if modelMilestones != lastModelMilestones:<EOL><INDENT>print("<STR_LIT>" % (<EOL>pprint.pformat(modelMilestones, indent=<NUM_LIT:4>)))<EOL>lastModelMilestones = modelMilestones<EOL><DEDENT>engStatus = jobInfo.getEngStatus()<EOL>if engStatus != lastEngStatus:<EOL><INDENT>print("<STR_LIT>" % (engStatus))<EOL>lastEngStatus = engStatus<EOL><DEDENT><DEDENT>if not hyperSearchFinished:<EOL><INDENT>if self._options["<STR_LIT>"] != None:<EOL><INDENT>if ((datetime.now() - lastUpdateTime) ><EOL>timedelta(minutes=self._options["<STR_LIT>"])):<EOL><INDENT>print("<STR_LIT>")<EOL>self.__cjDAO.jobCancel(jobID)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>time.sleep(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>modelIDs = self.__searchJob.queryModelIDs()<EOL>print("<STR_LIT>" % len(modelIDs))<EOL>print("<STR_LIT>")<EOL>jobInfo = self.__searchJob.getJobStatus(self._workers)<EOL>print("<STR_LIT>" % (jobInfo.getWorkerCompletionMsg()))<EOL>


return "<STR_LIT>" % (self.__class__.__name__, self.__nupicJobID)<EOL>


return tuple(self.__foundMetrcsKeySet)<EOL>

Returns a tuple of all metrics keys discovered while running HyperSearch.

        NOTE: This is an optimization so that our client may
            use this info for generating the report csv file without having
            to pre-scan all modelInfos

        Parameters:
        ----------------------------------------------------------------------
        retval:         Tuple of metrics keys discovered while running
                        HyperSearch;


params = self.__unwrapParams()<EOL>if "<STR_LIT>" in params:<EOL><INDENT>retval = dict()<EOL>queue = [(pair, retval) for pair in<EOL>params["<STR_LIT>"]["<STR_LIT>"].items()]<EOL>while len(queue) > <NUM_LIT:0>:<EOL><INDENT>pair, output = queue.pop()<EOL>k, v = pair<EOL>if ("<STR_LIT>" in v and "<STR_LIT>" in v and<EOL>"<STR_LIT>" in v):<EOL><INDENT>output[k] = v["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>if k not in output:<EOL><INDENT>output[k] = dict()<EOL><DEDENT>queue.extend((pair, output[k]) for pair in v.items())<EOL><DEDENT><DEDENT>return retval<EOL><DEDENT>

Parameters:
----------------------------------------------------------------------
retval:         a dictionary of model parameter labels. For each entry
                the key is the name of the parameter and the value
                is the value chosen for it.


return self.__rawInfo.numRecords<EOL>


assert not self.isWaitingToStart(), "<STR_LIT>" % self<EOL>return "<STR_LIT:%s>" % self.__rawInfo.startTime<EOL>

Returns model evaluation start time.

        NOTE: it's an error to call this method if isWaitingToStart() would
        return True.

        Parameters:
        ----------------------------------------------------------------------
        retval:         model evaluation start time


print("<STR_LIT>")<EOL>searchParams = _ClientJobUtils.makeSearchJobParamsDict(options=options)<EOL>

Prints a listing of experiments that would take place without
        actually executing them.

        Parameters:
        ----------------------------------------------------------------------
        options:        NupicRunPermutations options dict
        retval:         nothing


jobInfo = self.JobStatus(self.__nupicJobID, workers)<EOL>return jobInfo<EOL>

Parameters:
----------------------------------------------------------------------
workers:  If this job was launched outside of the nupic job engine, then this
           is an array of subprocess Popen instances, one for each worker
retval:         _NupicJob.JobStatus instance


def runWithConfig(swarmConfig, options,<EOL>outDir=None, outputLabel="<STR_LIT:default>",<EOL>permWorkDir=None, verbosity=<NUM_LIT:1>):

global g_currentVerbosityLevel<EOL>g_currentVerbosityLevel = verbosity<EOL>if outDir is None:<EOL><INDENT>outDir = os.getcwd()<EOL><DEDENT>if permWorkDir is None:<EOL><INDENT>permWorkDir = os.getcwd()<EOL><DEDENT>_checkOverwrite(options, outDir)<EOL>_generateExpFilesFromSwarmDescription(swarmConfig, outDir)<EOL>options["<STR_LIT>"] = swarmConfig<EOL>options["<STR_LIT>"] = outputLabel<EOL>options["<STR_LIT>"] = outDir<EOL>options["<STR_LIT>"] = permWorkDir<EOL>runOptions = _injectDefaultOptions(options)<EOL>_validateOptions(runOptions)<EOL>return _runAction(runOptions)<EOL>

Starts a swarm, given an dictionary configuration.
@param swarmConfig {dict} A complete [swarm description](http://nupic.docs.numenta.org/0.7.0.dev0/guides/swarming/running.html#the-swarm-description) object.
@param outDir {string} Optional path to write swarm details (defaults to
                       current working directory).
@param outputLabel {string} Optional label for output (defaults to "default").
@param permWorkDir {string} Optional location of working directory (defaults
                            to current working directory).
@param verbosity {int} Optional (1,2,3) increasing verbosity of output.

@returns {object} Model parameters


return cjdao.ClientJobsDAO.get()<EOL>

Returns: The shared cjdao.ClientJobsDAO instance


return self.__nupicJobID<EOL>

Semi-private method for retrieving the jobId

        Parameters:
        ----------------------------------------------------------------------
        retval:         Nupic Client JobID of this _NupicJob instance


self.__rawInfo = rawInfo<EOL>self.__cachedResults = None<EOL>assert self.__rawInfo.params is not None<EOL>self.__cachedParams = None<EOL>

Parameters:
----------------------------------------------------------------------
rawInfo:        A single model information element as returned by
                ClientJobsDAO.modelsInfo()
retval:         nothing.


return self.__rawInfo.genDescription<EOL>


self.__nupicJobID = nupicJobID<EOL>jobInfo = _clientJobsDB().jobInfo(nupicJobID)<EOL>assert jobInfo is not None, "<STR_LIT>" % nupicJobID<EOL>assert jobInfo.jobId == nupicJobID, "<STR_LIT>" % (jobInfo.jobId, nupicJobID)<EOL>_emit(Verbosity.DEBUG, "<STR_LIT>" % pprint.pformat(jobInfo, indent=<NUM_LIT:4>))<EOL>if jobInfo.params is not None:<EOL><INDENT>self.__params = json.loads(jobInfo.params)<EOL><DEDENT>else:<EOL><INDENT>self.__params = None<EOL><DEDENT>


def _launchWorkers(self, cmdLine, numWorkers):

self._workers = []<EOL>for i in range(numWorkers):<EOL><INDENT>stdout = tempfile.NamedTemporaryFile(delete=False)<EOL>stderr = tempfile.NamedTemporaryFile(delete=False)<EOL>p = subprocess.Popen(cmdLine, bufsize=<NUM_LIT:1>, env=os.environ, shell=True,<EOL>stdin=None, stdout=stdout, stderr=stderr)<EOL>p._stderr_file = stderr<EOL>p._stdout_file = stdout<EOL>self._workers.append(p)<EOL><DEDENT>

Launch worker processes to execute the given command line

        Parameters:
        -----------------------------------------------
        cmdLine: The command line for each worker
        numWorkers: number of workers to launch


parser = OptionParser(helpString)<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:int>", default=None,<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:str>", default=None,<EOL>help=("<STR_LIT>"<EOL>"<STR_LIT>"))<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:str>", default=None,<EOL>help=("<STR_LIT>"<EOL>"<STR_LIT>"))<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", default=None,<EOL>help="<STR_LIT>""<STR_LIT>""<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store_true>", default=False,<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store_true>", default=False,<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:int>", default=None,<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>(options, args) = parser.parse_args(argv[<NUM_LIT:1>:])<EOL>if len(args) != <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(args))<EOL><DEDENT>if (options.jobID and options.params):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if (options.jobID is None and options.params is None):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>initLogging(verbose=True)<EOL>hst = HypersearchWorker(options, argv[<NUM_LIT:1>:])<EOL>if options.params is None:<EOL><INDENT>try:<EOL><INDENT>jobID = hst.run()<EOL><DEDENT>except Exception as e:<EOL><INDENT>jobID = options.jobID<EOL>msg = io.StringIO()<EOL>print("<STR_LIT>" %(ErrorCodes.hypersearchLogicErr, e), file=msg)<EOL>traceback.print_exc(None, msg)<EOL>completionReason = ClientJobsDAO.CMPL_REASON_ERROR<EOL>completionMsg = msg.getvalue()<EOL>hst.logger.error(completionMsg)<EOL>jobsDAO = ClientJobsDAO.get()<EOL>workerCmpReason = jobsDAO.jobGetFields(options.jobID,<EOL>['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:<EOL><INDENT>jobsDAO.jobSetFields(options.jobID, fields=dict(<EOL>cancel=True,<EOL>workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,<EOL>workerCompletionMsg = completionMsg),<EOL>useConnectionID=False,<EOL>ignoreUnchanged=True)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>jobID = None<EOL>completionReason = ClientJobsDAO.CMPL_REASON_SUCCESS<EOL>completionMsg = "<STR_LIT>"<EOL>try:<EOL><INDENT>jobID = hst.run()<EOL><DEDENT>except Exception as e:<EOL><INDENT>jobID = hst._options.jobID<EOL>completionReason = ClientJobsDAO.CMPL_REASON_ERROR<EOL>completionMsg = "<STR_LIT>" % (e,)<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>if jobID is not None:<EOL><INDENT>cjDAO = ClientJobsDAO.get()<EOL>cjDAO.jobSetCompleted(jobID=jobID,<EOL>completionReason=completionReason,<EOL>completionMsg=completionMsg)<EOL><DEDENT><DEDENT><DEDENT>return jobID<EOL>

The main function of the HypersearchWorker script. This parses the command
line arguments, instantiates a HypersearchWorker instance, and then
runs it.

Parameters:
----------------------------------------------------------------------
retval:     jobID of the job we ran. This is used by unit test code
              when calling this working using the --params command
              line option (which tells this worker to insert the job
              itself).


<EOL>cjDAO = ClientJobsDAO.get()<EOL>(jobID, description) = cjDAO.modelsGetFields(<EOL>modelID,<EOL>["<STR_LIT>", "<STR_LIT>"])<EOL>(baseDescription,) = cjDAO.jobGetFields(jobID, ["<STR_LIT>"])<EOL>descriptionDirectory = tempfile.mkdtemp()<EOL>try:<EOL><INDENT>baseDescriptionFilePath = os.path.join(descriptionDirectory, "<STR_LIT>")<EOL>with open(baseDescriptionFilePath, mode="<STR_LIT:wb>") as f:<EOL><INDENT>f.write(baseDescription)<EOL><DEDENT>descriptionFilePath = os.path.join(descriptionDirectory, "<STR_LIT>")<EOL>with open(descriptionFilePath, mode="<STR_LIT:wb>") as f:<EOL><INDENT>f.write(description)<EOL><DEDENT>expIface = helpers.getExperimentDescriptionInterfaceFromModule(<EOL>helpers.loadExperimentDescriptionScriptFromDir(descriptionDirectory))<EOL>return json.dumps(<EOL>dict(<EOL>modelConfig=expIface.getModelDescription(),<EOL>inferenceArgs=expIface.getModelControl().get("<STR_LIT>", None)))<EOL><DEDENT>finally:<EOL><INDENT>shutil.rmtree(descriptionDirectory, ignore_errors=True)<EOL><DEDENT>

Retrieve the Engine-level model params from a Swarm model

    Args:
      modelID - Engine-level model ID of the Swarm model

    Returns:
      JSON-encoded string containing Model Params


def createAndStartSwarm(client, clientInfo="<STR_LIT>", clientKey="<STR_LIT>", params="<STR_LIT>",<EOL>minimumWorkers=None, maximumWorkers=None,<EOL>alreadyRunning=False):

if minimumWorkers is None:<EOL><INDENT>minimumWorkers = Configuration.getInt(<EOL>"<STR_LIT>")<EOL><DEDENT>if maximumWorkers is None:<EOL><INDENT>maximumWorkers = Configuration.getInt(<EOL>"<STR_LIT>")<EOL><DEDENT>return ClientJobsDAO.get().jobInsert(<EOL>client=client,<EOL>cmdLine="<STR_LIT>",<EOL>clientInfo=clientInfo,<EOL>clientKey=clientKey,<EOL>alreadyRunning=alreadyRunning,<EOL>params=params,<EOL>minimumWorkers=minimumWorkers,<EOL>maximumWorkers=maximumWorkers,<EOL>jobType=ClientJobsDAO.JOB_TYPE_HS)<EOL>

Create and start a swarm job.

    Args:
      client - A string identifying the calling client. There is a small limit
          for the length of the value. See ClientJobsDAO.CLIENT_MAX_LEN.
      clientInfo - JSON encoded dict of client specific information.
      clientKey - Foreign key. Limited in length, see ClientJobsDAO._initTables.
      params - JSON encoded dict of the parameters for the job. This can be
          fetched out of the database by the worker processes based on the jobID.
      minimumWorkers - The minimum workers to allocate to the swarm. Set to None
          to use the default.
      maximumWorkers - The maximum workers to allocate to the swarm. Set to None
          to use the swarm default. Set to 0 to use the maximum scheduler value.
      alreadyRunning - Insert a job record for an already running process. Used
          for testing.


if not options['<STR_LIT>'] ornot options['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>return None, None<EOL><DEDENT>predictedField = options['<STR_LIT>']['<STR_LIT>']<EOL>predictedFieldInfo = None<EOL>includedFields = options['<STR_LIT>']<EOL>for info in includedFields:<EOL><INDENT>if info['<STR_LIT>'] == predictedField:<EOL><INDENT>predictedFieldInfo = info<EOL>break<EOL><DEDENT><DEDENT>if predictedFieldInfo is None:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>" % predictedField<EOL>)<EOL><DEDENT>predictedFieldType = predictedFieldInfo['<STR_LIT>']<EOL>return predictedField, predictedFieldType<EOL>

Gets the predicted field and it's datatype from the options dictionary

    Returns: (predictedFieldName, predictedFieldType)


_metricSpecSchema = {'<STR_LIT>': {}}<EOL>results = []<EOL>for metric in options['<STR_LIT>']:<EOL><INDENT>for propertyName in list(_metricSpecSchema['<STR_LIT>'].keys()):<EOL><INDENT>_getPropertyValue(_metricSpecSchema, propertyName, metric)<EOL><DEDENT>specString, label = _generateMetricSpecString(<EOL>field=metric['<STR_LIT>'],<EOL>metric=metric['<STR_LIT>'],<EOL>params=metric['<STR_LIT>'],<EOL>inferenceElement=metric['<STR_LIT>'],<EOL>returnLabel=True)<EOL>if metric['<STR_LIT>']:<EOL><INDENT>options['<STR_LIT>'].append(label)<EOL><DEDENT>results.append(specString)<EOL><DEDENT>return results<EOL>

Generates the non-default metrics specified by the expGenerator params


return type(obj) in (str,)<EOL>

returns whether or not the object is a string


return "<STR_LIT>" % (errorString, usageString)<EOL>

Combines an error string and usage string into a regular format, so they
    all look consistent.


width = <NUM_LIT:7><EOL>fieldName = fieldInfo['<STR_LIT>']<EOL>fieldType = fieldInfo['<STR_LIT>']<EOL>encoderChoicesList = []<EOL>if fieldType in ['<STR_LIT:float>', '<STR_LIT:int>']:<EOL><INDENT>aggFunction = '<STR_LIT>'<EOL>encoders = [None]<EOL>for n in (<NUM_LIT>, <NUM_LIT:50>, <NUM_LIT>, <NUM_LIT>):<EOL><INDENT>encoder = dict(type='<STR_LIT>', name=fieldName, fieldname=fieldName,<EOL>n=n, w=width, clipInput=True,space="<STR_LIT>")<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoder['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoder['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoders.append(encoder)<EOL><DEDENT>encoderChoicesList.append(encoders)<EOL><DEDENT>elif fieldType == '<STR_LIT:string>':<EOL><INDENT>aggFunction = '<STR_LIT>'<EOL>encoders = [None]<EOL>encoder = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>, w=width)<EOL>encoders.append(encoder)<EOL>encoderChoicesList.append(encoders)<EOL><DEDENT>elif fieldType == '<STR_LIT>':<EOL><INDENT>aggFunction = '<STR_LIT>'<EOL>encoders = [None]<EOL>for radius in (<NUM_LIT:1>, <NUM_LIT:8>):<EOL><INDENT>encoder = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, timeOfDay=(width, radius))<EOL>encoders.append(encoder)<EOL><DEDENT>encoderChoicesList.append(encoders)<EOL>encoders = [None]<EOL>for radius in (<NUM_LIT:1>, <NUM_LIT:3>):<EOL><INDENT>encoder = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, dayOfWeek=(width, radius))<EOL>encoders.append(encoder)<EOL><DEDENT>encoderChoicesList.append(encoders)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (fieldType))<EOL><DEDENT>return (encoderChoicesList, aggFunction)<EOL>

Return a list of possible encoder parameter combinations for the given
    field and the default aggregation function to use. Each parameter combination
    is a dict defining the parameters for the encoder. Here is an example
    return value for the encoderChoicesList:

     [
       None,
       {'fieldname':'timestamp',
        'name': 'timestamp_timeOfDay',
        'type':'DateEncoder'
        'dayOfWeek': (7,1)
        },
       {'fieldname':'timestamp',
        'name': 'timestamp_timeOfDay',
        'type':'DateEncoder'
        'dayOfWeek': (7,3)
        },
    ],

    Parameters:
    --------------------------------------------------
    fieldInfo:      item from the 'includedFields' section of the
                      description JSON object

    retval:  (encoderChoicesList, aggFunction)
               encoderChoicesList: a list of encoder choice lists for this field.
                 Most fields will generate just 1 encoder choice list.
                 DateTime fields can generate 2 or more encoder choice lists,
                   one for dayOfWeek, one for timeOfDay, etc.
               aggFunction: name of aggregation function to use for this
                             field type


permStr = "<STR_LIT>"<EOL>if encoderDict.get('<STR_LIT>', False):<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if key == '<STR_LIT:n>' and encoderDict['<STR_LIT:type>'] != '<STR_LIT>':<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:w>"] + <NUM_LIT:7>,<EOL>encoderDict["<STR_LIT:w>"] + <NUM_LIT>)<EOL><DEDENT>else:<EOL><INDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>else:<EOL><INDENT>if encoderDict["<STR_LIT:type>"] in ["<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:type>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if key == "<STR_LIT:n>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:w>"] + <NUM_LIT:1>,<EOL>encoderDict["<STR_LIT:w>"] + <NUM_LIT>)<EOL><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>if value and not "<STR_LIT>" in encoderDict:<EOL><INDENT>permStr += "<STR_LIT>"% (_quoteAndEscape("<STR_LIT>"), _quoteAndEscape("<STR_LIT>"))<EOL><DEDENT>encoderDict.pop("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>elif encoderDict["<STR_LIT:type>"] in ["<STR_LIT>"]:<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:type>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>elif encoderDict["<STR_LIT:type>"] in ["<STR_LIT>"]:<EOL><INDENT>permStr = "<STR_LIT>"<EOL>for key, value in list(encoderDict.items()):<EOL><INDENT>if key == "<STR_LIT>":<EOL><INDENT>key = "<STR_LIT>"<EOL><DEDENT>elif key == "<STR_LIT:type>":<EOL><INDENT>continue<EOL><DEDENT>elif key == "<STR_LIT:name>":<EOL><INDENT>continue<EOL><DEDENT>if key == "<STR_LIT>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:type>"])<EOL>permStr += "<STR_LIT>"<EOL>permStr += "<STR_LIT>" % (value[<NUM_LIT:0>])<EOL><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:type>"])<EOL>permStr += "<STR_LIT>"<EOL>permStr += "<STR_LIT>" % (value[<NUM_LIT:0>])<EOL><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>permStr += "<STR_LIT>" % (encoderDict["<STR_LIT:type>"])<EOL>permStr += "<STR_LIT>"<EOL>permStr += "<STR_LIT>" % (value)<EOL><DEDENT>else:<EOL><INDENT>if issubclass(type(value), str):<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT>else:<EOL><INDENT>permStr += "<STR_LIT>" % (key, value)<EOL><DEDENT><DEDENT><DEDENT>permStr += "<STR_LIT:)>"<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" %(encoderDict["<STR_LIT:type>"]))<EOL><DEDENT><DEDENT>return permStr<EOL>

Generate the string that defines the permutations to apply for a given
    encoder.

    Parameters:
    -----------------------------------------------------------------------
    options: experiment params
    encoderDict: the encoder dict, which gets placed into the description.py

    For example, if the encoderDict contains:
        'consumption':     {
                  'clipInput': True,
                  'fieldname': u'consumption',
                  'n': 100,
                  'name': u'consumption',
                  'type': 'AdaptiveScalarEncoder',
                  'w': 21},

    The return string will contain:
      "PermuteEncoder(fieldName='consumption',
                      encoderClass='AdaptiveScalarEncoder',
                      w=21,
                      n=PermuteInt(28, 521),
                      clipInput=True)"


inferenceType = options['<STR_LIT>']<EOL>inferenceArgs = options['<STR_LIT>']<EOL>predictionSteps = inferenceArgs['<STR_LIT>']<EOL>metricWindow = options['<STR_LIT>']<EOL>if metricWindow is None:<EOL><INDENT>metricWindow = int(Configuration.get("<STR_LIT>"))<EOL><DEDENT>metricSpecStrings = []<EOL>optimizeMetricLabel = "<STR_LIT>"<EOL>metricSpecStrings.extend(_generateExtraMetricSpecs(options))<EOL>optimizeMetricSpec = None<EOL>if options['<STR_LIT>']:<EOL><INDENT>assert len(predictionSteps) == <NUM_LIT:1><EOL>predictionSteps = ['<STR_LIT>']<EOL><DEDENT>if inferenceType in (InferenceType.TemporalNextStep,<EOL>InferenceType.TemporalAnomaly,<EOL>InferenceType.TemporalMultiStep,<EOL>InferenceType.NontemporalMultiStep,<EOL>InferenceType.NontemporalClassification,<EOL>'<STR_LIT>'):<EOL><INDENT>predictedFieldName, predictedFieldType = _getPredictedField(options)<EOL>isCategory = _isCategory(predictedFieldType)<EOL>metricNames = ('<STR_LIT>',) if isCategory else ('<STR_LIT>', '<STR_LIT>')<EOL>trivialErrorMetric = '<STR_LIT>' if isCategory else '<STR_LIT>'<EOL>oneGramErrorMetric = '<STR_LIT>' if isCategory else '<STR_LIT>'<EOL>movingAverageBaselineName = '<STR_LIT>' if isCategory else '<STR_LIT>'<EOL>for metricName in metricNames:<EOL><INDENT>metricSpec, metricLabel =_generateMetricSpecString(field=predictedFieldName,<EOL>inferenceElement=InferenceElement.multiStepBestPredictions,<EOL>metric='<STR_LIT>',<EOL>params={'<STR_LIT>': metricName,<EOL>'<STR_LIT>':metricWindow,<EOL>'<STR_LIT>': predictionSteps},<EOL>returnLabel=True)<EOL>metricSpecStrings.append(metricSpec)<EOL><DEDENT>if options["<STR_LIT>"] is not None :<EOL><INDENT>metricParams = dict(options["<STR_LIT>"])<EOL>metricParams['<STR_LIT>'] = '<STR_LIT>'<EOL>metricParams['<STR_LIT>'] = predictionSteps<EOL>if not "<STR_LIT>" in metricParams:<EOL><INDENT>metricParams["<STR_LIT>"] = metricWindow<EOL><DEDENT>metricSpec, metricLabel =_generateMetricSpecString(field=predictedFieldName,<EOL>inferenceElement=InferenceElement.multiStepPredictions,<EOL>metric="<STR_LIT>",<EOL>params=metricParams,<EOL>returnLabel=True)<EOL>metricSpecStrings.append(metricSpec)<EOL><DEDENT>optimizeMetricSpec = metricSpec<EOL>metricLabel = metricLabel.replace('<STR_LIT:[>', '<STR_LIT>')<EOL>metricLabel = metricLabel.replace('<STR_LIT:]>', '<STR_LIT>')<EOL>optimizeMetricLabel = metricLabel<EOL>if options["<STR_LIT>"] is not None :<EOL><INDENT>optimizeMetricLabel = "<STR_LIT>"<EOL><DEDENT>if options["<STR_LIT>"]and inferenceType != InferenceType.NontemporalClassification:<EOL><INDENT>for steps in predictionSteps:<EOL><INDENT>metricSpecStrings.append(<EOL>_generateMetricSpecString(field=predictedFieldName,<EOL>inferenceElement=InferenceElement.prediction,<EOL>metric="<STR_LIT>",<EOL>params={'<STR_LIT>':metricWindow,<EOL>"<STR_LIT>":trivialErrorMetric,<EOL>'<STR_LIT>': steps})<EOL>)<EOL>if isCategory:<EOL><INDENT>metricSpecStrings.append(<EOL>_generateMetricSpecString(field=predictedFieldName,<EOL>inferenceElement=InferenceElement.prediction,<EOL>metric=movingAverageBaselineName,<EOL>params={'<STR_LIT>':metricWindow<EOL>,"<STR_LIT>":"<STR_LIT>",<EOL>"<STR_LIT>":<NUM_LIT:200>,<EOL>"<STR_LIT>": steps})<EOL>)<EOL><DEDENT>else :<EOL><INDENT>metricSpecStrings.append(<EOL>_generateMetricSpecString(field=predictedFieldName,<EOL>inferenceElement=InferenceElement.prediction,<EOL>metric=movingAverageBaselineName,<EOL>params={'<STR_LIT>':metricWindow<EOL>,"<STR_LIT>":"<STR_LIT>",<EOL>"<STR_LIT>":<NUM_LIT:200>,<EOL>"<STR_LIT>": steps})<EOL>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif inferenceType in (InferenceType.TemporalClassification):<EOL><INDENT>metricName = '<STR_LIT>'<EOL>trivialErrorMetric = '<STR_LIT>'<EOL>oneGramErrorMetric = '<STR_LIT>'<EOL>movingAverageBaselineName = '<STR_LIT>'<EOL>optimizeMetricSpec, optimizeMetricLabel =_generateMetricSpecString(inferenceElement=InferenceElement.classification,<EOL>metric=metricName,<EOL>params={'<STR_LIT>':metricWindow},<EOL>returnLabel=True)<EOL>metricSpecStrings.append(optimizeMetricSpec)<EOL>if options["<STR_LIT>"]:<EOL><INDENT>if inferenceType == InferenceType.TemporalClassification:<EOL><INDENT>metricSpecStrings.append(<EOL>_generateMetricSpecString(inferenceElement=InferenceElement.classification,<EOL>metric="<STR_LIT>",<EOL>params={'<STR_LIT>':metricWindow,<EOL>"<STR_LIT>":trivialErrorMetric})<EOL>)<EOL>metricSpecStrings.append(<EOL>_generateMetricSpecString(inferenceElement=InferenceElement.classification,<EOL>metric="<STR_LIT>",<EOL>params={'<STR_LIT>':metricWindow,<EOL>"<STR_LIT>":oneGramErrorMetric})<EOL>)<EOL>metricSpecStrings.append(<EOL>_generateMetricSpecString(inferenceElement=InferenceElement.classification,<EOL>metric=movingAverageBaselineName,<EOL>params={'<STR_LIT>':metricWindow<EOL>,"<STR_LIT>":"<STR_LIT>",<EOL>"<STR_LIT>":<NUM_LIT:200>})<EOL>)<EOL><DEDENT><DEDENT>if not options["<STR_LIT>"] == None :<EOL><INDENT>if not "<STR_LIT>" in options["<STR_LIT>"]:<EOL><INDENT>options["<STR_LIT>"]["<STR_LIT>"] = metricWindow<EOL><DEDENT>optimizeMetricSpec = _generateMetricSpecString(<EOL>inferenceElement=InferenceElement.classification,<EOL>metric="<STR_LIT>",<EOL>params=options["<STR_LIT>"])<EOL>optimizeMetricLabel = "<STR_LIT>"<EOL>metricSpecStrings.append(optimizeMetricSpec)<EOL><DEDENT><DEDENT>if options['<STR_LIT>']:<EOL><INDENT>for i in range(len(metricSpecStrings)):<EOL><INDENT>metricSpecStrings[i] = metricSpecStrings[i].replace(<EOL>"<STR_LIT>", "<STR_LIT>")<EOL><DEDENT>optimizeMetricLabel = optimizeMetricLabel.replace(<EOL>"<STR_LIT>", "<STR_LIT>")<EOL><DEDENT>return metricSpecStrings, optimizeMetricLabel<EOL>

Generates the Metrics for a given InferenceType

    Parameters:
    -------------------------------------------------------------------------
    options: ExpGenerator options
    retval: (metricsList, optimizeMetricLabel)
              metricsList: list of metric string names
              optimizeMetricLabel: Name of the metric which to optimize over


<EOL>installPath = os.path.dirname(__file__)<EOL>outputFile = open(outputFilePath, "<STR_LIT:w>")<EOL>outputLines = []<EOL>inputLines = []<EOL>firstFile = True<EOL>for templateFileName in templateFileNames:<EOL><INDENT>if not firstFile:<EOL><INDENT>inputLines.extend([os.linesep]*<NUM_LIT:2>)<EOL><DEDENT>firstFile = False<EOL>inputFilePath = os.path.join(installPath, templateFileName)<EOL>inputFile = open(inputFilePath)<EOL>inputLines.extend(inputFile.readlines())<EOL>inputFile.close()<EOL><DEDENT>print("<STR_LIT>", len(inputLines), "<STR_LIT>")<EOL>for line in inputLines:<EOL><INDENT>tempLine = line<EOL>for k, v in replacementDict.items():<EOL><INDENT>if v is None:<EOL><INDENT>v = "<STR_LIT:None>"<EOL><DEDENT>tempLine = re.sub(k, v, tempLine)<EOL><DEDENT>outputFile.write(tempLine)<EOL><DEDENT>outputFile.close()<EOL>

Generates a file by applying token replacements to the given template
    file

    templateFileName:
                    A list of template file names; these files are assumed to be in
                    the same directory as the running experiment_generator.py script.
                    ExpGenerator will perform the substitution and concanetate
                    the files in the order they are specified

    outputFilePath: Absolute path of the output file

    replacementDict:
                    A dictionary of token/replacement pairs


def _generateMetricSpecString(inferenceElement, metric,<EOL>params=None, field=None,<EOL>returnLabel=False):

metricSpecArgs = dict(metric=metric,<EOL>field=field,<EOL>params=params,<EOL>inferenceElement=inferenceElement)<EOL>metricSpecAsString = "<STR_LIT>" %'<STR_LIT:U+002CU+0020>'.join(['<STR_LIT>' % (item[<NUM_LIT:0>],item[<NUM_LIT:1>])<EOL>for item in metricSpecArgs.items()])<EOL>if not returnLabel:<EOL><INDENT>return metricSpecAsString<EOL><DEDENT>spec = MetricSpec(**metricSpecArgs)<EOL>metricLabel = spec.getLabel()<EOL>return metricSpecAsString, metricLabel<EOL>

Generates the string representation of a MetricSpec object, and returns
    the metric key associated with the metric.

    Parameters:
    -----------------------------------------------------------------------
    inferenceElement:
      An InferenceElement value that indicates which part of the inference this
      metric is computed on

    metric:
      The type of the metric being computed (e.g. aae, avg_error)

    params:
      A dictionary of parameters for the metric. The keys are the parameter names
      and the values should be the parameter values (e.g. window=200)

    field:
      The name of the field for which this metric is being computed

    returnLabel:
      If True, returns the label of the MetricSpec that was generated


assert _isString(string)<EOL>return pprint.pformat(string)<EOL>

string:   input string (ascii or unicode)

Returns:  a quoted string with characters that are represented in python via
          escape sequences converted to those escape sequences


def _getPropertyValue(schema, propertyName, options):

if propertyName not in options:<EOL><INDENT>paramsSchema = schema['<STR_LIT>'][propertyName]<EOL>if '<STR_LIT:default>' in paramsSchema:<EOL><INDENT>options[propertyName] = paramsSchema['<STR_LIT:default>']<EOL><DEDENT>else:<EOL><INDENT>options[propertyName] = None<EOL><DEDENT><DEDENT>

Checks to see if property is specified in 'options'. If not, reads the
    default value from the schema


xInt = int(round(x))<EOL>return (abs(x - xInt) < precision * x, xInt)<EOL>

Return (isInt, intValue) for a given floating point number.

Parameters:
----------------------------------------------------------------------
x:  floating point number to evaluate
precision: desired precision
retval:   (isInt, intValue)
          isInt: True if x is close enough to an integer value
          intValue: x as an integer


width = <NUM_LIT><EOL>encoderDictsList = []<EOL>if options['<STR_LIT>'] in ["<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"]:<EOL><INDENT>classifierOnlyField = options['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>classifierOnlyField = None<EOL><DEDENT>for fieldInfo in includedFields:<EOL><INDENT>fieldName = fieldInfo['<STR_LIT>']<EOL>fieldType = fieldInfo['<STR_LIT>']<EOL>if fieldType in ['<STR_LIT:float>', '<STR_LIT:int>']:<EOL><INDENT>runDelta = fieldInfo.get("<STR_LIT>", False)<EOL>if runDelta or "<STR_LIT>" in fieldInfo:<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>, w=width, clipInput=True)<EOL>if runDelta:<EOL><INDENT>encoderDict["<STR_LIT>"] = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>, w=width, clipInput=True)<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if ('<STR_LIT>' in fieldInfo and '<STR_LIT>' in fieldInfo)and (encoderDict['<STR_LIT:type>'] == '<STR_LIT>'):<EOL><INDENT>encoderDict['<STR_LIT:type>'] = '<STR_LIT>'<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL><DEDENT>elif fieldType == '<STR_LIT:string>':<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name=fieldName,<EOL>fieldname=fieldName, n=<NUM_LIT:100>+width, w=width)<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL><DEDENT>elif fieldType == '<STR_LIT>':<EOL><INDENT>encoderDict = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, timeOfDay=(width, <NUM_LIT:1>))<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL>encoderDict = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, dayOfWeek=(width, <NUM_LIT:1>))<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL>encoderDict = dict(type='<STR_LIT>', name='<STR_LIT>' % (fieldName),<EOL>fieldname=fieldName, weekend=(width))<EOL>if '<STR_LIT>' in fieldInfo:<EOL><INDENT>encoderDict['<STR_LIT:type>'] = fieldInfo['<STR_LIT>']<EOL><DEDENT>encoderDictsList.append(encoderDict)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (fieldType))<EOL><DEDENT>if fieldName == classifierOnlyField:<EOL><INDENT>clEncoderDict = dict(encoderDict)<EOL>clEncoderDict['<STR_LIT>'] = True<EOL>clEncoderDict['<STR_LIT:name>'] = '<STR_LIT>'<EOL>encoderDictsList.append(clEncoderDict)<EOL>if options["<STR_LIT>"]["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>encoderDictsList.remove(encoderDict)<EOL><DEDENT><DEDENT><DEDENT>if options.get('<STR_LIT>') is not None:<EOL><INDENT>tempList=[]<EOL>for encoderDict in encoderDictsList:<EOL><INDENT>if encoderDict['<STR_LIT:name>'] in options['<STR_LIT>']:<EOL><INDENT>tempList.append(encoderDict)<EOL><DEDENT><DEDENT>encoderDictsList = tempList<EOL><DEDENT>encoderSpecsList = []<EOL>permEncoderChoicesList = []<EOL>for encoderDict in encoderDictsList:<EOL><INDENT>if encoderDict['<STR_LIT:name>'].find('<STR_LIT:\\>') >= <NUM_LIT:0>:<EOL><INDENT>raise _ExpGeneratorException("<STR_LIT>")<EOL><DEDENT>for c in _ILLEGAL_FIELDNAME_CHARACTERS:<EOL><INDENT>if encoderDict['<STR_LIT:name>'].find(c) >= <NUM_LIT:0>:<EOL><INDENT>raise _ExpGeneratorException("<STR_LIT>"  %(c, encoderDict['<STR_LIT:name>']))<EOL><DEDENT><DEDENT>constructorStr = _generatePermEncoderStr(options, encoderDict)<EOL>encoderKey = _quoteAndEscape(encoderDict['<STR_LIT:name>'])<EOL>encoderSpecsList.append("<STR_LIT>" % (<EOL>encoderKey,<EOL><NUM_LIT:2>*_ONE_INDENT,<EOL>pprint.pformat(encoderDict, indent=<NUM_LIT:2>*_INDENT_STEP)))<EOL>permEncoderChoicesList.append("<STR_LIT>" % (encoderKey, constructorStr))<EOL><DEDENT>encoderSpecsStr = '<STR_LIT>'.join(encoderSpecsList)<EOL>permEncoderChoicesStr = '<STR_LIT:\n>'.join(permEncoderChoicesList)<EOL>permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, <NUM_LIT:1>,<EOL>indentFirstLine=True)<EOL>return (encoderSpecsStr, permEncoderChoicesStr)<EOL>

Generate and return the following encoder related substitution variables:

    encoderSpecsStr:
      For the base description file, this string defines the default
      encoding dicts for each encoder. For example:

           __gym_encoder = {   'fieldname': 'gym',
            'n': 13,
            'name': 'gym',
            'type': 'SDRCategoryEncoder',
            'w': 7},
          __address_encoder = {   'fieldname': 'address',
            'n': 13,
            'name': 'address',
            'type': 'SDRCategoryEncoder',
            'w': 7}

    permEncoderChoicesStr:
      For the permutations file, this defines the possible
      encoder dicts for each encoder. For example:

          '__gym_encoder' : PermuteEncoder('gym', 'SDRCategoryEncoder', w=7,
              n=100),

          '__address_encoder' : PermuteEncoder('address', 'SDRCategoryEncoder',
                w=7, n=100),

          '__timestamp_dayOfWeek_encoder' : PermuteEncoder('timestamp',
              'DateEncoder.timeOfDay', w=7, radius=PermuteChoices([1, 8])),

          '__consumption_encoder': PermuteEncoder('consumption', 'AdaptiveScalarEncoder',
              w=7, n=PermuteInt(13, 500, 20), minval=0,
              maxval=PermuteInt(100, 300, 25)),

    Parameters:
    --------------------------------------------------
    includedFields:  item from the 'includedFields' section of the
                      description JSON object. This is a list of dicts, each
                      dict defining the field name, type, and optional min
                      and max values.

    retval:  (encoderSpecsStr permEncoderChoicesStr)


def _handleDescriptionOption(cmdArgStr, outDir, usageStr, hsVersion,<EOL>claDescriptionTemplateFile):

<EOL>try:<EOL><INDENT>args = json.loads(cmdArgStr)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise _InvalidCommandArgException(<EOL>_makeUsageErrorStr(<EOL>("<STR_LIT>" +"<STR_LIT>") % (str(e), cmdArgStr), usageStr))<EOL><DEDENT>filesDescription = _generateExperiment(args, outDir, hsVersion=hsVersion,<EOL>claDescriptionTemplateFile = claDescriptionTemplateFile)<EOL>pprint.pprint(filesDescription)<EOL>return<EOL>

Parses and validates the --description option args and executes the
request

Parameters:
-----------------------------------------------------------------------
cmdArgStr:  JSON string compatible with _gExperimentDescriptionSchema
outDir:     where to place generated experiment files
usageStr:   program usage string
hsVersion:  which version of hypersearch permutations file to generate, can
              be 'v1' or 'v2'
claDescriptionTemplateFile: Filename containing the template description
retval:     nothing


coordinates = [<NUM_LIT:0>] * len(dimensions)<EOL>shifted = index<EOL>for i in xrange(len(dimensions) - <NUM_LIT:1>, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>coordinates[i] = shifted % dimensions[i]<EOL>shifted = shifted / dimensions[i]<EOL><DEDENT>coordinates[<NUM_LIT:0>] = shifted<EOL>return coordinates<EOL>

Translate an index into coordinates, using the given coordinate system.

Similar to ``numpy.unravel_index``.

:param index: (int) The index of the point. The coordinates are expressed as a
       single index by using the dimensions as a mixed radix definition. For
       example, in dimensions 42x10, the point [1, 4] is index
       1*420 + 4*10 = 460.

:param dimensions (list of ints) The coordinate system.

:returns: (list) of coordinates of length ``len(dimensions)``.


def wrappingNeighborhood(centerIndex, radius, dimensions):

centerPosition = coordinatesFromIndex(centerIndex, dimensions)<EOL>intervals = []<EOL>for i, dimension in enumerate(dimensions):<EOL><INDENT>left = centerPosition[i] - radius<EOL>right = min(centerPosition[i] + radius,<EOL>left + dimensions[i] - <NUM_LIT:1>)<EOL>interval = [v % dimension for v in xrange(left, right + <NUM_LIT:1>)]<EOL>intervals.append(interval)<EOL><DEDENT>coords = numpy.array(list(itertools.product(*intervals)))<EOL>return numpy.ravel_multi_index(coords.T, dimensions)<EOL>

Like :meth:`neighborhood`, except that the neighborhood isn't truncated when
it's near an edge. It wraps around to the other side.

:param centerIndex: (int) The index of the point. The coordinates are
       expressed as a single index by using the dimensions as a mixed radix
       definition. For example, in dimensions 42x10, the point [1, 4] is index
       1*420 + 4*10 = 460.

:param radius: (int) The radius of this neighborhood about the
       ``centerIndex``.

:param dimensions: (indexable sequence) The dimensions of the world outside
       this neighborhood.

:returns: (numpy array) The points in the neighborhood, including
          ``centerIndex``.


curLambda = current + self.offset<EOL>x, logProb = PoissonDistribution(curLambda).sample(r)<EOL>logBackward = PoissonDistribution(x+self.offset).logDensity(current)<EOL>return x, logProb, logBackward<EOL>

Generates a random sample from the Poisson probability distribution with
        with location and scale parameter equal to the current value (passed in).
        Returns the value of the random sample, the log of the probability of
        sampling that value, and the log of the probability of sampling the current
        value if the roles of the new sample and the current sample were reversed
        (the log of the backward proposal probability).


stay = (r.uniform(<NUM_LIT:0>, <NUM_LIT:1>) < self.kernel)<EOL>if stay:<EOL><INDENT>logKernel = numpy.log(self.kernel)<EOL>return current, logKernel, logKernel<EOL><DEDENT>else: <EOL><INDENT>curIndex = self.keyMap[current]<EOL>ri = r.randint(<NUM_LIT:0>, self.nKeys-<NUM_LIT:1>)<EOL>logKernel = numpy.log(<NUM_LIT:1.0> - self.kernel)<EOL>lp = logKernel + self.logp<EOL>if ri < curIndex: return self.keys[ri], lp, lp<EOL>else: return self.keys[ri+<NUM_LIT:1>], lp, lp<EOL><DEDENT>

Generates a random sample from the discrete probability distribution and
        returns its value, the log of the probability of sampling that value and the
        log of the probability of sampling the current value (passed in).


lx = numpy.asarray(lx)<EOL>base = lx.max()<EOL>x = numpy.exp(lx - base)<EOL>result = x / x.sum()<EOL>conventional = (numpy.exp(lx) / numpy.exp(lx).sum())<EOL>assert similar(result, conventional)<EOL>return result<EOL>

Accepts log-values as input, exponentiates them,
normalizes and returns the result.
Handles underflow by rescaling so that the largest values is exactly 1.0.


if out is None:<EOL><INDENT>out = numpy.zeros(A.shape)<EOL><DEDENT>indicator1 = A >= B<EOL>assert indicator1.all(), "<STR_LIT>"<EOL>out[indicator1] = A[indicator1] + numpy.log(<NUM_LIT:1> - numpy.exp(B[indicator1]-A[indicator1]))<EOL>return out<EOL>

returns log(exp(A) - exp(B)). A and B are numpy arrays. values in A should be
    greater than or equal to corresponding values in B


lx = numpy.asarray(lx)<EOL>base = lx.max()<EOL>if numpy.isinf(base):<EOL><INDENT>return base<EOL><DEDENT>if (epsilon is not None) and (base < epsilon):<EOL><INDENT>return epsilon<EOL><DEDENT>x = numpy.exp(lx - base)<EOL>ssum = x.sum()<EOL>result = numpy.log(ssum) + base<EOL>return result<EOL>

Accepts log-values as input, exponentiates them, computes the sum,
then converts the sum back to log-space and returns the result.
Handles underflow by rescaling so that the largest values is exactly 1.0.


lx = numpy.asarray(lx)<EOL>base = lx.max()<EOL>x = numpy.exp(lx - base)<EOL>ssum = x.sum(<NUM_LIT:0>)<EOL>normalized = nsum0(lx)<EOL>result = numpy.log(normalized)<EOL>conventional = numpy.log(numpy.exp(lx).sum(<NUM_LIT:0>) / numpy.exp(lx).sum())<EOL>assert similar(result, conventional)<EOL>return result<EOL>

Accepts log-values as input, exponentiates them, sums down the rows
(first dimension), normalizes, then converts the sum back to
log-space and returns the result.
Handles underflow by rescaling so that the largest values is exactly 1.0.


m = x.max()<EOL>return (i for i, v in enumerate(x) if v == m)<EOL>

Get tuple (actually a generator) of indices where the max value of
array x occurs. Requires that x have a max() method, as x.max()
(in the case of NumPy) is much faster than max(x).
For a simpler, faster argmax when there is only a single maximum entry,
or when knowing only the first index where the maximum occurs,
call argmax() on a NumPy array.

:param x: Any sequence that has a max() method.
:returns: Generator with the indices where the max value occurs.


if self.hist_: return self.hist_.nRows()<EOL>else: return <NUM_LIT:0><EOL>

Gets the number of rows in the histogram.

        :returns: Integer number of rows.


x = numpy.zeros(size, dtype=dtype)<EOL>if hasattr(pos, '<STR_LIT>'):<EOL><INDENT>for i in pos: x[i] = <NUM_LIT:1><EOL><DEDENT>else: x[pos] = <NUM_LIT:1><EOL>return x<EOL>

Returns an array of length size and type dtype that is everywhere 0,
except in the indices listed in sequence pos.

:param pos:   A single integer or sequence of integers that specify
       the position of ones to be set.
:param size:  The total size of the array to be returned.
:param dtype: The element type (compatible with NumPy array())
       of the array to be returned.
:returns: An array of length size and element type dtype.


x = numpy.zeros(size, dtype=dtype)<EOL>x[pos] = <NUM_LIT:1><EOL>return x<EOL>

Returns an array of length size and type dtype that is everywhere 0,
except in the index in pos.

:param pos: (int) specifies the position of the one entry that will be set.
:param size: (int) The total size of the array to be returned.
:param dtype: The element type (compatible with NumPy array())
       of the array to be returned.
:returns: (list) of length ``size`` and element type ``dtype``.


m = self.hist_.toDense()<EOL>for j in xrange(m.shape[<NUM_LIT:1>]): <EOL><INDENT>cmax = m[:,j].max()<EOL>if cmax:<EOL><INDENT>m[:,j] = numpy.array(m[:,j] == cmax, dtype=dtype)<EOL><DEDENT><DEDENT>self.hack_ = SparseMatrix(<NUM_LIT:0>, self.hist_.nCols())<EOL>for i in xrange(m.shape[<NUM_LIT:0>]):<EOL><INDENT>self.hack_.addRow(m[i,:])<EOL><DEDENT>

Hack to act like clean_outcpd on zeta1.TopLevelNode.
        Take the max element in each to column, set it to 1, and set all the
        other elements to 0.
        Only called by inferRowMaxProd() and only needed if an updateRow()
        has been called since the last clean_outcpd().


x = rgen.poisson(self.lambdaParameter)<EOL>return x, self.logDensity(x)<EOL>

Generates a random sample from the Poisson probability distribution and
        returns its value and the log of the probability of sampling that value.


return lgamma(x + <NUM_LIT:1.0>)<EOL>

Approximation to the log of the factorial function.


rf = rgen.uniform(<NUM_LIT:0>, self.sum)<EOL>index = bisect.bisect(self.cdf, rf)<EOL>return self.keys[index], numpy.log(self.pmf[index])<EOL>

Generates a random sample from the discrete probability distribution
        and returns its value and the log of the probability of sampling that value.


<EOL>if x.shape[<NUM_LIT:0>] != y.shape[<NUM_LIT:0>]:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (x.shape, y.shape))<EOL><DEDENT>if x.shape[<NUM_LIT:0>] < <NUM_LIT:2>:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % x.shape)<EOL><DEDENT>order = np.argsort(x)<EOL>x = x[order]<EOL>y = y[order]<EOL>h = np.diff(x)<EOL>area = np.sum(h * (y[<NUM_LIT:1>:] + y[:-<NUM_LIT:1>])) / <NUM_LIT><EOL>return area<EOL>

Compute Area Under the Curve (AUC) using the trapezoidal rule

    Parameters
    ----------
    x : array, shape = [n]
        x coordinates

    y : array, shape = [n]
        y coordinates

    Returns
    -------
    auc : float

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import metrics
    >>> y = np.array([1, 1, 2, 2])
    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred)
    >>> metrics.auc(fpr, tpr)
    0.75


keys = keywords.keys()<EOL>sequences = [keywords[key] for key in keys]<EOL>wheels = map(iter, sequences)<EOL>digits = [it.next( ) for it in wheels]<EOL>while True:<EOL><INDENT>yield dict(zip(keys, digits))<EOL>for i in range(len(digits)-<NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>try:<EOL><INDENT>digits[i] = wheels[i].next( )<EOL>break<EOL><DEDENT>except StopIteration:<EOL><INDENT>wheels[i] = iter(sequences[i])<EOL>digits[i] = wheels[i].next( )<EOL><DEDENT><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Similar to cross(), but generates output dictionaries instead of tuples.


<EOL>wheels = map(iter, sequences)<EOL>digits = [it.next( ) for it in wheels]<EOL>while True:<EOL><INDENT>yield tuple(digits)<EOL>for i in range(len(digits)-<NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>try:<EOL><INDENT>digits[i] = wheels[i].next( )<EOL>break<EOL><DEDENT>except StopIteration:<EOL><INDENT>wheels[i] = iter(sequences[i])<EOL>digits[i] = wheels[i].next( )<EOL><DEDENT><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>


newAverage, self.slidingWindow, self.total = self.compute(<EOL>self.slidingWindow, self.total, newValue, self.windowSize)<EOL>return newAverage<EOL>


try:<EOL><INDENT>line = next(f)<EOL>assert line.endswith('<STR_LIT>')<EOL>valid_times = list(range(<NUM_LIT>))<EOL>t = <NUM_LIT:0> <EOL>club = None<EOL>clubName = None<EOL>lastDate = None<EOL>while True:<EOL><INDENT>assert t in valid_times<EOL>consumption = <NUM_LIT:0><EOL>for x in range(<NUM_LIT:4>):<EOL><INDENT>line = f.next()[:-<NUM_LIT:1>]<EOL>fields = line.split('<STR_LIT:U+002C>')<EOL>assert len(fields) == <NUM_LIT:4><EOL>for i, field in enumerate(fields):<EOL><INDENT>assert field[<NUM_LIT:0>] == '<STR_LIT:">' and field[-<NUM_LIT:1>] == '<STR_LIT:">'<EOL>fields[i] = field[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL><DEDENT>name = fields[<NUM_LIT:1>]<EOL>partialNames = ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>for pn in partialNames:<EOL><INDENT>if pn in name:<EOL><INDENT>name = pn<EOL><DEDENT><DEDENT>if name != clubName:<EOL><INDENT>clubName = name<EOL>club = clubs[name]<EOL><DEDENT>tokens = fields[<NUM_LIT:2>].split()<EOL>if len(tokens) == <NUM_LIT:1>:<EOL><INDENT>assert consumption == <NUM_LIT:0> and t == <NUM_LIT:0><EOL><DEDENT>date = tokens[<NUM_LIT:0>]<EOL>consumption += float(fields[<NUM_LIT:3>])<EOL><DEDENT>club.updateRecord(date, t, consumption)<EOL>t += <NUM_LIT:1><EOL>t %= <NUM_LIT><EOL><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>return<EOL><DEDENT>

Process the consumption a club

    - Skip the header line
    - Iterate over lines
      - Read 4 records at a time
        - Parse each line: club, date, time, consumption
        - Get club object from dictionary if needed
        - Aggregate consumption
      - Call club.processConsumption() with data


<EOL>initProb = numpy.zeros(numCategories)<EOL>initProb[<NUM_LIT:0>] = <NUM_LIT:0.5><EOL>initProb[<NUM_LIT:4>] = <NUM_LIT:0.5><EOL>firstOrder = dict()<EOL>for catIdx in range(numCategories):<EOL><INDENT>key = str([catIdx])<EOL>probs = numpy.ones(numCategories) / numCategories<EOL>if catIdx == <NUM_LIT:0> or catIdx == <NUM_LIT:4>:<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT:1>] = <NUM_LIT:1.0>    <EOL><DEDENT>firstOrder[key] = probs<EOL><DEDENT>secondOrder = dict()<EOL>for firstIdx in range(numCategories):<EOL><INDENT>for secondIdx in range(numCategories):<EOL><INDENT>key = str([firstIdx, secondIdx])<EOL>probs = numpy.ones(numCategories) / numCategories<EOL>if key == str([<NUM_LIT:0>,<NUM_LIT:1>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT:2>] = <NUM_LIT>   <EOL>probs[<NUM_LIT:3>] = <NUM_LIT>   <EOL><DEDENT>elif key == str([<NUM_LIT:4>,<NUM_LIT:1>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT:2>] = <NUM_LIT>   <EOL>probs[<NUM_LIT:3>] = <NUM_LIT>   <EOL><DEDENT>secondOrder[key] = probs<EOL><DEDENT><DEDENT>return (initProb, firstOrder, secondOrder, <NUM_LIT:3>)<EOL>

Generate the initial, first order, and second order transition
    probabilities for 'model0'. For this model, we generate the following
    set of sequences:

    1-2-3   (4X)
    1-2-4   (1X)
    5-2-3   (1X)
    5-2-4   (4X)

    Parameters:
    ----------------------------------------------------------------------
    numCategories:      Number of categories
    retval: (initProb, firstOrder, secondOrder, seqLen)
              initProb:     Initial probability for each category. This is a vector
                              of length len(categoryList).
              firstOrder:   A dictionary of the 1st order probabilities. The key
                              is the 1st element of the sequence, the value is
                              the probability of each 2nd element given the first.
              secondOrder:  A dictionary of the 2nd order probabilities. The key
                              is the first 2 elements of the sequence, the value is
                              the probability of each possible 3rd element given the
                              first two.
              seqLen:       Desired length of each sequence. The 1st element will
                            be generated using the initProb, the 2nd element by the
                            firstOrder table, and the 3rd and all successive
                            elements by the secondOrder table.

    Here is an example of some return values:
    initProb:         [0.7, 0.2, 0.1]

    firstOrder:       {'[0]': [0.3, 0.3, 0.4],
                       '[1]': [0.3, 0.3, 0.4],
                       '[2]': [0.3, 0.3, 0.4]}

    secondOrder:      {'[0,0]': [0.3, 0.3, 0.4],
                       '[0,1]': [0.3, 0.3, 0.4],
                       '[0,2]': [0.3, 0.3, 0.4],
                       '[1,0]': [0.3, 0.3, 0.4],
                       '[1,1]': [0.3, 0.3, 0.4],
                       '[1,2]': [0.3, 0.3, 0.4],
                       '[2,0]': [0.3, 0.3, 0.4],
                       '[2,1]': [0.3, 0.3, 0.4],
                       '[2,2]': [0.3, 0.3, 0.4]}


<EOL>initProb = numpy.zeros(numCategories)<EOL>initProb[<NUM_LIT:0>] = <NUM_LIT:0.5><EOL>initProb[<NUM_LIT:1>] = <NUM_LIT:0.5><EOL>firstOrder = dict()<EOL>for catIdx in range(numCategories):<EOL><INDENT>key = str([catIdx])<EOL>probs = numpy.ones(numCategories) / numCategories<EOL>if catIdx == <NUM_LIT:0> or catIdx == <NUM_LIT:1>:<EOL><INDENT>indices = numpy.array([<NUM_LIT:10>,<NUM_LIT:11>,<NUM_LIT:12>,<NUM_LIT>,<NUM_LIT>])<EOL>probs.fill(<NUM_LIT:0>)<EOL>probs[indices] = <NUM_LIT:1.0>    <EOL>probs /= probs.sum()<EOL><DEDENT>firstOrder[key] = probs<EOL><DEDENT>secondOrder = dict()<EOL>for firstIdx in range(numCategories):<EOL><INDENT>for secondIdx in range(numCategories):<EOL><INDENT>key = str([firstIdx, secondIdx])<EOL>probs = numpy.ones(numCategories) / numCategories<EOL>if key == str([<NUM_LIT:0>,<NUM_LIT:10>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT:15>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:0>,<NUM_LIT:11>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT:16>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:0>,<NUM_LIT:12>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:0>,<NUM_LIT>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:0>,<NUM_LIT>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:1>,<NUM_LIT:10>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT:20>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:1>,<NUM_LIT:11>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:1>,<NUM_LIT:12>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:1>,<NUM_LIT>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>elif key == str([<NUM_LIT:1>,<NUM_LIT>]):<EOL><INDENT>probs.fill(<NUM_LIT:0>)<EOL>probs[<NUM_LIT>] = <NUM_LIT:1><EOL><DEDENT>secondOrder[key] = probs<EOL><DEDENT><DEDENT>return (initProb, firstOrder, secondOrder, <NUM_LIT:3>)<EOL>

Generate the initial, first order, and second order transition
    probabilities for 'model1'. For this model, we generate the following
    set of sequences:

    0-10-15 (1X)
    0-11-16 (1X)
    0-12-17 (1X)
    0-13-18 (1X)
    0-14-19 (1X)

    1-10-20 (1X)
    1-11-21 (1X)
    1-12-22 (1X)
    1-13-23 (1X)
    1-14-24 (1X)

    Parameters:
    ----------------------------------------------------------------------
    numCategories:      Number of categories
    retval: (initProb, firstOrder, secondOrder, seqLen)
              initProb:     Initial probability for each category. This is a vector
                              of length len(categoryList).
              firstOrder:   A dictionary of the 1st order probabilities. The key
                              is the 1st element of the sequence, the value is
                              the probability of each 2nd element given the first.
              secondOrder:  A dictionary of the 2nd order probabilities. The key
                              is the first 2 elements of the sequence, the value is
                              the probability of each possible 3rd element given the
                              first two.
              seqLen:       Desired length of each sequence. The 1st element will
                            be generated using the initProb, the 2nd element by the
                            firstOrder table, and the 3rd and all successive
                            elements by the secondOrder table.

    Here is an example of some return values:
    initProb:         [0.7, 0.2, 0.1]

    firstOrder:       {'[0]': [0.3, 0.3, 0.4],
                       '[1]': [0.3, 0.3, 0.4],
                       '[2]': [0.3, 0.3, 0.4]}

    secondOrder:      {'[0,0]': [0.3, 0.3, 0.4],
                       '[0,1]': [0.3, 0.3, 0.4],
                       '[0,2]': [0.3, 0.3, 0.4],
                       '[1,0]': [0.3, 0.3, 0.4],
                       '[1,1]': [0.3, 0.3, 0.4],
                       '[1,2]': [0.3, 0.3, 0.4],
                       '[2,0]': [0.3, 0.3, 0.4],
                       '[2,1]': [0.3, 0.3, 0.4],
                       '[2,2]': [0.3, 0.3, 0.4]}


<EOL>print("<STR_LIT>" % (filename))<EOL>numRecords, numFields = data.shape<EOL>fields = [('<STR_LIT>'%(i+<NUM_LIT:1>), '<STR_LIT:float>', '<STR_LIT>') for i in range(numFields)]<EOL>outFile = File(filename, fields)<EOL>for i in range(numRecords):<EOL><INDENT>outFile.write(data[i].tolist())<EOL><DEDENT>outFile.close()<EOL>


return TMRegionProto<EOL>


self._sequencePos = <NUM_LIT:0><EOL>self._fpLogTPOutput = None<EOL>self.logPathOutput = None<EOL>

Initialize all ephemerals used by derived classes.


typeNames = {int: '<STR_LIT>', float: '<STR_LIT>', str: '<STR_LIT>', bool: '<STR_LIT:bool>', tuple: '<STR_LIT>'}<EOL>def getArgType(arg):<EOL><INDENT>t = typeNames.get(type(arg), '<STR_LIT>')<EOL>count = <NUM_LIT:0> if t == '<STR_LIT>' else <NUM_LIT:1><EOL>if t == '<STR_LIT>':<EOL><INDENT>t = typeNames.get(type(arg[<NUM_LIT:0>]), '<STR_LIT>')<EOL>count = len(arg)<EOL><DEDENT>if t == '<STR_LIT:bool>':<EOL><INDENT>t = '<STR_LIT>'<EOL><DEDENT>return (t, count)<EOL><DEDENT>def getConstraints(arg):<EOL><INDENT>t = typeNames.get(type(arg), '<STR_LIT>')<EOL>if t == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>elif t == '<STR_LIT:bool>':<EOL><INDENT>return '<STR_LIT:bool>'<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT><DEDENT>TemporalClass = _getTPClass(temporalImp)<EOL>tArgTuples = _buildArgs(TemporalClass.__init__)<EOL>temporalSpec = {}<EOL>for argTuple in tArgTuples:<EOL><INDENT>d = dict(<EOL>description=argTuple[<NUM_LIT:1>],<EOL>accessMode='<STR_LIT>',<EOL>dataType=getArgType(argTuple[<NUM_LIT:2>])[<NUM_LIT:0>],<EOL>count=getArgType(argTuple[<NUM_LIT:2>])[<NUM_LIT:1>],<EOL>constraints=getConstraints(argTuple[<NUM_LIT:2>]))<EOL>temporalSpec[argTuple[<NUM_LIT:0>]] = d<EOL><DEDENT>temporalSpec.update(dict(<EOL>columnCount=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>cellsPerColumn=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>inputWidth=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>predictedSegmentDecrement=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>orColumnOutputs=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT:bool>'),<EOL>cellsSavePath=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>temporalImp=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>))<EOL>otherSpec = dict(<EOL>learningMode=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>defaultValue=True,<EOL>constraints='<STR_LIT:bool>'),<EOL>inferenceMode=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>defaultValue=False,<EOL>constraints='<STR_LIT:bool>'),<EOL>computePredictedActiveCellIndices=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>defaultValue=False,<EOL>constraints='<STR_LIT:bool>'),<EOL>anomalyMode=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>defaultValue=False,<EOL>constraints='<STR_LIT:bool>'),<EOL>topDownMode=dict(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>defaultValue=False,<EOL>constraints='<STR_LIT:bool>'),<EOL>activeOutputCount=dict(<EOL>description='<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT>'),<EOL>storeDenseOutput=dict(<EOL>description="""<STR_LIT>""",<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:1>,<EOL>constraints='<STR_LIT:bool>'),<EOL>logPathOutput=dict(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>',<EOL>accessMode='<STR_LIT>',<EOL>dataType='<STR_LIT>',<EOL>count=<NUM_LIT:0>,<EOL>constraints='<STR_LIT>'),<EOL>)<EOL>return temporalSpec, otherSpec<EOL>

Build the additional specs in three groups (for the inspector)

    Use the type of the default argument to set the Spec type, defaulting
    to 'Byte' for None and complex types

    Determines the spatial parameters based on the selected implementation.
    It defaults to TemporalMemory.
    Determines the temporal parameters based on the temporalImp


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.storeDenseOutput = False<EOL><DEDENT>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.computePredictedActiveCellIndices = False<EOL><DEDENT>self.__dict__.update(state)<EOL>self._loaded = True<EOL>self._initialize()<EOL>

Set the state of ourself from a serialized state.


instance = cls(proto.columnCount, proto.inputWidth, proto.cellsPerColumn)<EOL>instance.temporalImp = proto.temporalImp<EOL>instance.learningMode = proto.learningMode<EOL>instance.inferenceMode = proto.inferenceMode<EOL>instance.anomalyMode = proto.anomalyMode<EOL>instance.topDownMode = proto.topDownMode<EOL>instance.computePredictedActiveCellIndices = (<EOL>proto.computePredictedActiveCellIndices)<EOL>instance.orColumnOutputs = proto.orColumnOutputs<EOL>if instance.temporalImp == "<STR_LIT>":<EOL><INDENT>tmProto = proto.backtrackingTM<EOL><DEDENT>elif instance.temporalImp == "<STR_LIT>":<EOL><INDENT>tmProto = proto.backtrackingTMCpp<EOL><DEDENT>elif instance.temporalImp == "<STR_LIT>":<EOL><INDENT>tmProto = proto.temporalMemory<EOL><DEDENT>elif instance.temporalImp == "<STR_LIT>":<EOL><INDENT>tmProto = proto.temporalMemory<EOL><DEDENT>else:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>".format(<EOL>instance.temporalImp))<EOL><DEDENT>instance._tfdr = _getTPClass(proto.temporalImp).read(tmProto)<EOL>return instance<EOL>

Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.readFromProto`.

Read state from proto object.

:param proto: TMRegionProto capnproto object


Doesn't include the spatial, temporal and other parameters

:returns: (dict) the base Spec for TMRegion.


<EOL>if self._tfdr is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>self._conditionalBreak()<EOL>self._iterations += <NUM_LIT:1><EOL>buInputVector = inputs['<STR_LIT>']<EOL>resetSignal = False<EOL>if '<STR_LIT>' in inputs:<EOL><INDENT>assert len(inputs['<STR_LIT>']) == <NUM_LIT:1><EOL>if inputs['<STR_LIT>'][<NUM_LIT:0>] != <NUM_LIT:0>:<EOL><INDENT>self._tfdr.reset()<EOL>self._sequencePos = <NUM_LIT:0>  <EOL><DEDENT><DEDENT>if self.computePredictedActiveCellIndices:<EOL><INDENT>prevPredictedState = self._tfdr.getPredictedState().reshape(-<NUM_LIT:1>).astype('<STR_LIT>')<EOL><DEDENT>if self.anomalyMode:<EOL><INDENT>prevPredictedColumns = self._tfdr.topDownCompute().copy().nonzero()[<NUM_LIT:0>]<EOL><DEDENT>tpOutput = self._tfdr.compute(buInputVector, self.learningMode, self.inferenceMode)<EOL>self._sequencePos += <NUM_LIT:1><EOL>if self.orColumnOutputs:<EOL><INDENT>tpOutput= tpOutput.reshape(self.columnCount,<EOL>self.cellsPerColumn).max(axis=<NUM_LIT:1>)<EOL><DEDENT>if self._fpLogTPOutput:<EOL><INDENT>output = tpOutput.reshape(-<NUM_LIT:1>)<EOL>outputNZ = tpOutput.nonzero()[<NUM_LIT:0>]<EOL>outStr = "<STR_LIT:U+0020>".join(["<STR_LIT>" % int(token) for token in outputNZ])<EOL>print(output.size, outStr, file=self._fpLogTPOutput)<EOL><DEDENT>outputs['<STR_LIT>'][:] = tpOutput.flat<EOL>if self.topDownMode:<EOL><INDENT>outputs['<STR_LIT>'][:] = self._tfdr.topDownCompute().copy()<EOL><DEDENT>if self.anomalyMode:<EOL><INDENT>activeLearnCells = self._tfdr.getLearnActiveStateT()<EOL>size = activeLearnCells.shape[<NUM_LIT:0>] * activeLearnCells.shape[<NUM_LIT:1>]<EOL>outputs['<STR_LIT>'][:] = activeLearnCells.reshape(size)<EOL>activeColumns = buInputVector.nonzero()[<NUM_LIT:0>]<EOL>outputs['<STR_LIT>'][:] = anomaly.computeRawAnomalyScore(<EOL>activeColumns, prevPredictedColumns)<EOL><DEDENT>if self.computePredictedActiveCellIndices:<EOL><INDENT>activeState = self._tfdr._getActiveState().reshape(-<NUM_LIT:1>).astype('<STR_LIT>')<EOL>activeIndices = numpy.where(activeState != <NUM_LIT:0>)[<NUM_LIT:0>]<EOL>predictedIndices= numpy.where(prevPredictedState != <NUM_LIT:0>)[<NUM_LIT:0>]<EOL>predictedActiveIndices = numpy.intersect1d(activeIndices, predictedIndices)<EOL>outputs["<STR_LIT>"].fill(<NUM_LIT:0>)<EOL>outputs["<STR_LIT>"][activeIndices] = <NUM_LIT:1><EOL>outputs["<STR_LIT>"].fill(<NUM_LIT:0>)<EOL>outputs["<STR_LIT>"][predictedActiveIndices] = <NUM_LIT:1><EOL><DEDENT>


return self._getEphemeralMembersBase() + self._getEphemeralMembers()<EOL>

Returns a concatenated list of both the standard base class
ephemeral members, as well as any additional ephemeral members
(e.g., file handles, etc.).


spec = cls.getBaseSpec()<EOL>t, o = _getAdditionalSpecs(temporalImp=gDefaultTemporalImp)<EOL>spec['<STR_LIT>'].update(t)<EOL>spec['<STR_LIT>'].update(o)<EOL>return spec<EOL>

Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.getSpec`.

The parameters collection is constructed based on the parameters specified
by the various components (spatialSpec, temporalSpec and otherSpec)


if temporalImp == '<STR_LIT>':<EOL><INDENT>return backtracking_tm.BacktrackingTM<EOL><DEDENT>elif temporalImp == '<STR_LIT>':<EOL><INDENT>return backtracking_tm_cpp.BacktrackingTMCPP<EOL><DEDENT>elif temporalImp == '<STR_LIT>':<EOL><INDENT>return backtracking_tm_shim.TMShim<EOL><DEDENT>elif temporalImp == '<STR_LIT>':<EOL><INDENT>return backtracking_tm_shim.TMCPPShim<EOL><DEDENT>elif temporalImp == '<STR_LIT>':<EOL><INDENT>return backtracking_tm_shim.MonitoredTMShim<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % (temporalImp))<EOL><DEDENT>

Return the class corresponding to the given temporalImp string


def getParameterArray(self, name, index, a):

p = self.getParameter(name)<EOL>if (not hasattr(p, '<STR_LIT>')):<EOL><INDENT>raise Exception("<STR_LIT>" % name)<EOL><DEDENT>if len(p) >  <NUM_LIT:0>:<EOL><INDENT>a[:] = p[:]<EOL><DEDENT>


return [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>]<EOL>


def getParameter(self, name, index=-<NUM_LIT:1>):

<EOL>return PyRegion.getParameter(self, name, index)<EOL>


if self._sdrClassifier is None:<EOL><INDENT>self._sdrClassifier = SDRClassifierFactory.create(<EOL>steps=self.stepsList,<EOL>alpha=self.alpha,<EOL>verbosity=self.verbosity,<EOL>implementation=self.implementation,<EOL>)<EOL><DEDENT>

Overrides :meth:`nupic.bindings.regions.PyRegion.PyRegion.initialize`.

Is called once by NuPIC before the first call to compute().
Initializes self._sdrClassifier if it is not already initialized.


return self._sdrClassifier<EOL>