print()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>if env is not None and len(env) > <NUM_LIT:0>:<EOL><INDENT>envItems = []<EOL>for (key, value) in env.items():<EOL><INDENT>if (sys.platform.startswith('<STR_LIT>')):<EOL><INDENT>envItems.append("<STR_LIT>" % (key, value))<EOL><DEDENT>else:<EOL><INDENT>envItems.append("<STR_LIT>" % (key, value))<EOL><DEDENT><DEDENT>if (sys.platform.startswith('<STR_LIT>')):<EOL><INDENT>envStr = "<STR_LIT>" % ('<STR_LIT>'.join(envItems))<EOL><DEDENT>else:<EOL><INDENT>envStr = "<STR_LIT>" % ('<STR_LIT:;>'.join(envItems))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>envStr = '<STR_LIT>'<EOL><DEDENT>cmdLine = '<STR_LIT>''<STR_LIT>'% (envStr, loggingLevel)<EOL>cjDAO = ClientJobsDAO.get()<EOL>jobID = cjDAO.jobInsert(client='<STR_LIT:test>', cmdLine=cmdLine,<EOL>params=json.dumps(jobParams),<EOL>minimumWorkers=<NUM_LIT:1>, maximumWorkers=maxNumWorkers,<EOL>jobType = cjDAO.JOB_TYPE_HS)<EOL>workerCmdLine = '<STR_LIT>''<STR_LIT>'% (envStr, jobID, loggingLevel)<EOL>workers = self._launchWorkers(cmdLine=workerCmdLine, numWorkers=maxNumWorkers)<EOL>print("<STR_LIT>" % (jobID))<EOL>print("<STR_LIT>" % (maxNumWorkers),cmdLine)<EOL>if not waitForCompletion:<EOL><INDENT>return (jobID, None, None)<EOL><DEDENT>if timeoutSec is None:<EOL><INDENT>timeout=DEFAULT_JOB_TIMEOUT_SEC<EOL><DEDENT>else:<EOL><INDENT>timeout=timeoutSec<EOL><DEDENT>startTime = time.time()<EOL>lastUpdate = time.time()<EOL>lastCompleted = <NUM_LIT:0><EOL>lastCompletedWithError = <NUM_LIT:0><EOL>lastCompletedAsOrphan = <NUM_LIT:0><EOL>lastStarted = <NUM_LIT:0><EOL>lastJobStatus = "<STR_LIT>"<EOL>lastJobResults = None<EOL>lastActiveSwarms = None<EOL>lastEngStatus = None<EOL>modelIDs = []<EOL>print("<STR_LIT>" % ("<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>"))<EOL>print("<STR_LIT>")<EOL>while (lastJobStatus != ClientJobsDAO.STATUS_COMPLETED)and (time.time() - lastUpdate < timeout):<EOL><INDENT>printUpdate = False<EOL>if g_myEnv.options.verbosity == <NUM_LIT:0>:<EOL><INDENT>time.sleep(<NUM_LIT:0.5>)<EOL><DEDENT>jobInfo = self._getJobInfo(cjDAO, workers, jobID)<EOL>if jobInfo.status != lastJobStatus:<EOL><INDENT>if jobInfo.status == ClientJobsDAO.STATUS_RUNNINGand lastJobStatus != ClientJobsDAO.STATUS_RUNNING:<EOL><INDENT>print("<STR_LIT>"% (jobInfo.jobId))<EOL><DEDENT>lastJobStatus = jobInfo.status<EOL>printUpdate = True<EOL><DEDENT>if g_myEnv.options.verbosity >= <NUM_LIT:1>:<EOL><INDENT>if jobInfo.engWorkerState is not None:<EOL><INDENT>activeSwarms = json.loads(jobInfo.engWorkerState)['<STR_LIT>']<EOL>if activeSwarms != lastActiveSwarms:<EOL><INDENT>print("<STR_LIT>", '<STR_LIT>'.join(activeSwarms))<EOL>lastActiveSwarms = activeSwarms<EOL>print()<EOL><DEDENT><DEDENT>if jobInfo.results != lastJobResults:<EOL><INDENT>print("<STR_LIT>", jobInfo.results, "<STR_LIT>")<EOL>lastJobResults = jobInfo.results<EOL><DEDENT>if jobInfo.engStatus != lastEngStatus:<EOL><INDENT>print('<STR_LIT>' % jobInfo.engStatus)<EOL>print()<EOL>lastEngStatus = jobInfo.engStatus<EOL><DEDENT><DEDENT>modelCounters = cjDAO.modelsGetUpdateCounters(jobID)<EOL>if len(modelCounters) != lastStarted:<EOL><INDENT>modelIDs = [x.modelId for x in modelCounters]<EOL>lastStarted = len(modelCounters)<EOL>printUpdate = True<EOL><DEDENT>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>completed = <NUM_LIT:0><EOL>completedWithError = <NUM_LIT:0><EOL>completedAsOrphan = <NUM_LIT:0><EOL>infos = cjDAO.modelsGetResultAndStatus(modelIDs)<EOL>for info in infos:<EOL><INDENT>if info.status == ClientJobsDAO.STATUS_COMPLETED:<EOL><INDENT>completed += <NUM_LIT:1><EOL>if info.completionReason == ClientJobsDAO.CMPL_REASON_ERROR:<EOL><INDENT>completedWithError += <NUM_LIT:1><EOL><DEDENT>if info.completionReason == ClientJobsDAO.CMPL_REASON_ORPHAN:<EOL><INDENT>completedAsOrphan += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>if completed != lastCompletedor completedWithError != lastCompletedWithErroror completedAsOrphan != lastCompletedAsOrphan:<EOL><INDENT>lastCompleted = completed<EOL>lastCompletedWithError = completedWithError<EOL>lastCompletedAsOrphan = completedAsOrphan<EOL>printUpdate = True<EOL><DEDENT><DEDENT>if printUpdate:<EOL><INDENT>lastUpdate = time.time()<EOL>if g_myEnv.options.verbosity >= <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL><DEDENT>print("<STR_LIT>" % (lastJobStatus, lastStarted,<EOL>lastCompleted,<EOL>lastCompletedWithError,<EOL>lastCompletedAsOrphan))<EOL><DEDENT><DEDENT>print("<STR_LIT>" % (lastJobStatus, lastStarted,<EOL>lastCompleted,<EOL>lastCompletedWithError,<EOL>lastCompletedAsOrphan))<EOL>jobInfo = self._getJobInfo(cjDAO, workers, jobID)<EOL>if not ignoreErrModels:<EOL><INDENT>self.assertEqual (jobInfo.completionReason,<EOL>ClientJobsDAO.CMPL_REASON_SUCCESS)<EOL><DEDENT>models = cjDAO.modelsGetUpdateCounters(jobID)<EOL>modelIDs = [model.modelId for model in models]<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>results = cjDAO.modelsGetResultAndStatus(modelIDs)<EOL><DEDENT>else:<EOL><INDENT>results = []<EOL><DEDENT>metricResults = []<EOL>for result in results:<EOL><INDENT>if result.results is not None:<EOL><INDENT>metricResults.append(list(json.loads(result.results)[<NUM_LIT:1>].values())[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>metricResults.append(None)<EOL><DEDENT>if not ignoreErrModels:<EOL><INDENT>self.assertNotEqual(result.completionReason, cjDAO.CMPL_REASON_ERROR,<EOL>"<STR_LIT>" % (result.completionMsg))<EOL><DEDENT><DEDENT>return (jobID, jobInfo, results, metricResults)<EOL>

Given a prepared, filled in jobParams for a hypersearch, this starts
        the job, waits for it to complete, and returns the results for all
        models.

        Parameters:
        -------------------------------------------------------------------
        jobParams:        filled in job params for a hypersearch
        loggingLevel:    logging level to use in the Hypersearch worker
        maxNumWorkers:    max # of worker processes to use
        env:             if not None, this is a dict of environment variables
                            that should be sent to each worker process. These can
                            aid in re-using the same description/permutations file
                            for different tests.
        waitForCompletion: If True, wait for job to complete before returning
                           If False, then return resultsInfoForAllModels and
                           metricResults will be None
        ignoreErrModels:  If true, ignore erred models
        retval:          (jobID, jobInfo, resultsInfoForAllModels, metricResults)


pass<EOL>

Method called to prepare the test fixture. This is called by the
        unittest framework immediately before calling the test method; any exception
        raised by this method will be considered an error rather than a test
        failure. The default implementation does nothing.


def _runPermutationsLocal(self, jobParams, loggingLevel=logging.INFO,<EOL>env=None, waitForCompletion=True,<EOL>continueJobId=None, ignoreErrModels=False):

print()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>if env is not None:<EOL><INDENT>saveEnvState = copy.deepcopy(os.environ)<EOL>os.environ.update(env)<EOL><DEDENT>cjDAO = ClientJobsDAO.get()<EOL>if continueJobId is None:<EOL><INDENT>jobID = cjDAO.jobInsert(client='<STR_LIT:test>', cmdLine='<STR_LIT>',<EOL>params=json.dumps(jobParams),<EOL>alreadyRunning=True, minimumWorkers=<NUM_LIT:1>, maximumWorkers=<NUM_LIT:1>,<EOL>jobType = cjDAO.JOB_TYPE_HS)<EOL><DEDENT>else:<EOL><INDENT>jobID = continueJobId<EOL><DEDENT>args = ['<STR_LIT>', '<STR_LIT>' % (jobID),<EOL>'<STR_LIT>' % (loggingLevel)]<EOL>if continueJobId is None:<EOL><INDENT>args.append('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>hypersearch_worker.main(args)<EOL><DEDENT>except SystemExit:<EOL><INDENT>pass<EOL><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT>if env is not None:<EOL><INDENT>os.environ = saveEnvState<EOL><DEDENT>models = cjDAO.modelsGetUpdateCounters(jobID)<EOL>modelIDs = [model.modelId for model in models]<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>results = cjDAO.modelsGetResultAndStatus(modelIDs)<EOL><DEDENT>else:<EOL><INDENT>results = []<EOL><DEDENT>metricResults = []<EOL>for result in results:<EOL><INDENT>if result.results is not None:<EOL><INDENT>metricResults.append(list(json.loads(result.results)[<NUM_LIT:1>].values())[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>metricResults.append(None)<EOL><DEDENT>if not ignoreErrModels:<EOL><INDENT>self.assertNotEqual(result.completionReason, cjDAO.CMPL_REASON_ERROR,<EOL>"<STR_LIT>" % (result.completionMsg))<EOL><DEDENT><DEDENT>jobInfo = cjDAO.jobInfo(jobID)<EOL>return (jobID, jobInfo, results, metricResults)<EOL>

This runs permutations on the given experiment using just 1 worker
        in the current process

        Parameters:
        -------------------------------------------------------------------
        jobParams:        filled in job params for a hypersearch
        loggingLevel:    logging level to use in the Hypersearch worker
        env:             if not None, this is a dict of environment variables
                            that should be sent to each worker process. These can
                            aid in re-using the same description/permutations file
                            for different tests.
        waitForCompletion: If True, wait for job to complete before returning
                           If False, then return resultsInfoForAllModels and
                           metricResults will be None
        continueJobId:    If not None, then this is the JobId of a job we want
                          to continue working on with another worker.
        ignoreErrModels:  If true, ignore erred models
        retval:          (jobId, jobInfo, resultsInfoForAllModels, metricResults)


def _getJobInfo(self, cjDAO, workers, jobID):

<EOL>jobInfo = cjDAO.jobInfo(jobID)<EOL>runningCount = <NUM_LIT:0><EOL>for worker in workers:<EOL><INDENT>retCode = worker.poll()<EOL>if retCode is None:<EOL><INDENT>runningCount += <NUM_LIT:1><EOL><DEDENT><DEDENT>if runningCount > <NUM_LIT:0>:<EOL><INDENT>status = ClientJobsDAO.STATUS_RUNNING<EOL><DEDENT>else:<EOL><INDENT>status = ClientJobsDAO.STATUS_COMPLETED<EOL><DEDENT>jobInfo = jobInfo._replace(status=status)<EOL>if status == ClientJobsDAO.STATUS_COMPLETED:<EOL><INDENT>jobInfo = jobInfo._replace(<EOL>completionReason=ClientJobsDAO.CMPL_REASON_SUCCESS)<EOL><DEDENT>return jobInfo<EOL>

Return the job info for a job

        Parameters:
        -----------------------------------------------
        cjDAO:   client jobs database instance
        workers: list of workers for this job
        jobID:   which job ID

        retval: job info


<EOL>self.resetExtraLogItems()<EOL>

Method called immediately after the test method has been called and the
        result recorded. This is called even if the test method raised an exception,
        so the implementation in subclasses may need to be particularly careful
        about checking internal state. Any exception raised by this method will be
        considered an error rather than a test failure. This method will only be
        called if the setUp() succeeds, regardless of the outcome of the test
        method. The default implementation does nothing.


cjDAO = ClientJobsDAO.get()<EOL>jobResults = cjDAO.jobGetFields(jobID, ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>print("<STR_LIT>" % (jobResults,))<EOL>jobResults = json.loads(jobResults)<EOL>return jobResults['<STR_LIT>']<EOL>

Parameters:
-------------------------------------------------------------------
jobID:            jobID of successfully-completed Hypersearch job

retval:           modelID of the winning model


_debugOut(("<STR_LIT>") %(args,))<EOL>p = subprocess.Popen(args,<EOL>env=os.environ,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL>_debugOut(("<STR_LIT>") % (args,))<EOL>(stdoutData, stderrData) = p.communicate()<EOL>_debugOut(("<STR_LIT>" +"<STR_LIT>") %(args, p.returncode, type(stdoutData), stdoutData, stderrData))<EOL>result = dict(<EOL>exitStatus = p.returncode,<EOL>stdoutData = stdoutData,<EOL>stderrData = stderrData,<EOL>)<EOL>_debugOut(("<STR_LIT>") %(args, pprint.pformat(result, indent=<NUM_LIT:4>)))<EOL>return result<EOL>

args:     Args list as defined for the args parameter in subprocess.Popen()

Returns:  result dicionary:
            {
              'exitStatus':<exit-status-of-external-command>,
              'stdoutData':"string",
              'stderrData':"string"
            }


print("<STR_LIT>")<EOL>print("<STR_LIT>" % (self.__class__, self._testMethodName))<EOL>


helpString ="""<STR_LIT>"""<EOL>allTests = _getTestList()<EOL>for test in allTests:<EOL><INDENT>helpString += "<STR_LIT>" % (test)<EOL><DEDENT>parser = OptionParser(helpString,conflict_handler="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", default=<NUM_LIT:0>, type="<STR_LIT:int>",<EOL>help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store_true>", default=False,<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", action="<STR_LIT:store>", type="<STR_LIT:int>",<EOL>default=logging.INFO,<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", dest="<STR_LIT>", default=<NUM_LIT:2>, type='<STR_LIT:int>',<EOL>help=("<STR_LIT>"<EOL>"<STR_LIT>"))<EOL>return parser.parse_args(args=cls.args)<EOL>

Parse our command-line args/options and strip them from sys.argv
Returns the tuple (parsedOptions, remainingArgs)


return cls._processArgs()[<NUM_LIT:1>]<EOL>

Consumes the test arguments and returns the remaining arguments meant
        for unittest.man


return None<EOL>

Override to force unittest framework to use test method names instead
        of docstrings in the report.


newcounter = <NUM_LIT:0><EOL>for job in self.__procs:<EOL><INDENT>if job.is_alive():<EOL><INDENT>newcounter+=<NUM_LIT:1><EOL><DEDENT><DEDENT>self.__numRunningProcs = newcounter<EOL>return newcounter<EOL>

Function that iterates through the running Processes
        and counts the number of processes that are currently alive.
        Sets numRunningProcs to this count


jobsrunning = self.__numRunningProcs<EOL>if(maxJobs > <NUM_LIT:1>):<EOL><INDENT>jobsindx = <NUM_LIT:0><EOL>while(jobsindx<len(self.testQ) or jobsrunning><NUM_LIT:0>):<EOL><INDENT>if(jobsindx<len(self.testQ) and jobsrunning<maxJobs):<EOL><INDENT>curJob = self.testQ[jobsindx]<EOL>p = Process(target = curJob[<NUM_LIT:0>], args = curJob[<NUM_LIT:1>])<EOL>p.start()<EOL>self.__procs.append(p)<EOL>jobsindx+=<NUM_LIT:1><EOL><DEDENT>if jobsrunning >= maxJobs:<EOL><INDENT>time.sleep(<NUM_LIT:30>)<EOL>print ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif jobsindx == len(self.testQ):<EOL><INDENT>time.sleep(<NUM_LIT:30>)<EOL>print("<STR_LIT>")<EOL><DEDENT>jobsrunning = self.__updateProcessCounter()<EOL>for proc in self.__procs:<EOL><INDENT>if proc.exitcode == <NUM_LIT:1>:<EOL><INDENT>self.cancelJobs()<EOL>assert False, ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>while True:<EOL><INDENT>result = self.__resultQ.get(True, <NUM_LIT:5>)<EOL>self.assertBenchmarks(result)<EOL><DEDENT><DEDENT>except Empty:<EOL><INDENT>pass<EOL><DEDENT>

Function that launched Hypersearch benchmark jobs.
        Runs jobs contained in self.testQ, until maxJobs are running
        in parallel at which point it waits until some jobs finish.


print("<STR_LIT>")<EOL>for proc in self.__procs:<EOL><INDENT>if not proc.is_alive():<EOL><INDENT>proc.terminate()<EOL><DEDENT><DEDENT>print("<STR_LIT>")<EOL>

Function that cancels all the jobs in the
        process queue.


<EOL>dataPath = os.path.join(self.datadir, "<STR_LIT>", "<STR_LIT>")<EOL>streamDef = dict(<EOL>version=<NUM_LIT:1>,<EOL>info="<STR_LIT>",<EOL>streams=[<EOL>dict(source="<STR_LIT>" % (dataPath),<EOL>info="<STR_LIT>",<EOL>columns=["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>last_record=self.splits['<STR_LIT>'],)<EOL>],<EOL>aggregation={<EOL>'<STR_LIT>' : <NUM_LIT:1>,<EOL>'<STR_LIT>' : [<EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>]<EOL>},<EOL>)<EOL>expDesc = OPFBenchmarkRunner.EXP_COMMON.copy()<EOL>expDesc["<STR_LIT>"]["<STR_LIT>"] = "<STR_LIT>"<EOL>expDesc.update({<EOL>"<STR_LIT>": streamDef,<EOL>"<STR_LIT>": [<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>"<EOL>},<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:float>",<EOL>"<STR_LIT>":  <NUM_LIT>,<EOL>"<STR_LIT>":  <NUM_LIT>,<EOL>},<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:string>",<EOL>},<EOL>],<EOL>"<STR_LIT>": self.__recordsToProcess,<EOL>})<EOL>expdir = os.path.join(self.outdir,  "<STR_LIT>")<EOL>self.generateModules(expDesc,  expdir)<EOL>self.descriptions["<STR_LIT>"]=(expdir, expDesc)<EOL>return expdir<EOL>

Try running a basic experiment and permutations.


<EOL>dataPath = os.path.join(self.datadir, "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>")<EOL>streamDef = dict(<EOL>version=<NUM_LIT:1>,<EOL>info="<STR_LIT>",<EOL>streams=[<EOL>dict(source="<STR_LIT>" % (dataPath),<EOL>info="<STR_LIT>",<EOL>columns=["<STR_LIT>","<STR_LIT>"],<EOL>last_record=self.splits['<STR_LIT>'],),<EOL>],<EOL>)<EOL>expDesc = OPFBenchmarkRunner.EXP_COMMON.copy()<EOL>expDesc["<STR_LIT>"]["<STR_LIT>"] = "<STR_LIT>"<EOL>expDesc.update({<EOL>"<STR_LIT>": streamDef,<EOL>"<STR_LIT>": [<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:int>",<EOL>"<STR_LIT>":  -<NUM_LIT:10>,<EOL>"<STR_LIT>":  <NUM_LIT>,<EOL>},<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:int>",<EOL>"<STR_LIT>":  -<NUM_LIT:10>,<EOL>"<STR_LIT>":  <NUM_LIT>,<EOL>},<EOL>],<EOL>"<STR_LIT>": self.__recordsToProcess,<EOL>})<EOL>expdir = os.path.join(self.outdir,  "<STR_LIT>")<EOL>self.generateModules(expDesc,  expdir)<EOL>self.descriptions["<STR_LIT>"]=(expdir, expDesc)<EOL>return expdir<EOL>

Try running a basic experiment and permutations


ret = []<EOL>if self.maxBranchings is None:<EOL><INDENT>self.maxBranchings = [None]<EOL><DEDENT>else:<EOL><INDENT>self.maxBranchings = self.maxBranchings.split('<STR_LIT:U+002C>')<EOL><DEDENT>if self.maxParticles is None:<EOL><INDENT>self.maxParticles = [None]<EOL><DEDENT>else:<EOL><INDENT>self.maxParticles = self.maxParticles.split("<STR_LIT:U+002C>")<EOL><DEDENT>for branch in self.maxBranchings:<EOL><INDENT>for part in self.maxParticles:<EOL><INDENT>curdict = dict()<EOL>if not branch is None:<EOL><INDENT>curdict[self.BRANCHING_PROP] = branch<EOL><DEDENT>if not part is None:<EOL><INDENT>curdict[self.PARTICLE_PROP] = part<EOL><DEDENT>ret+=[curdict]<EOL><DEDENT><DEDENT>return ret<EOL>

Setup up a dict of branchings and particles


<EOL>dataPath = os.path.join(self.datadir, "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>")<EOL>streamDef = dict(<EOL>version=<NUM_LIT:1>,<EOL>info="<STR_LIT>",<EOL>streams=[<EOL>dict(source="<STR_LIT>" % (dataPath),<EOL>info="<STR_LIT>",<EOL>columns=["<STR_LIT>","<STR_LIT>"],<EOL>last_record=self.splits['<STR_LIT>']),<EOL>],<EOL>)<EOL>expDesc = OPFBenchmarkRunner.EXP_COMMON.copy()<EOL>expDesc["<STR_LIT>"]["<STR_LIT>"] = "<STR_LIT>"<EOL>expDesc.update({<EOL>"<STR_LIT>": streamDef,<EOL>"<STR_LIT>": [<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:int>",<EOL>"<STR_LIT>":  -<NUM_LIT:10>,<EOL>"<STR_LIT>":  <NUM_LIT>,<EOL>},<EOL>{ "<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:int>",<EOL>"<STR_LIT>":  -<NUM_LIT:10>,<EOL>"<STR_LIT>":  <NUM_LIT>,<EOL>}<EOL>],<EOL>"<STR_LIT>": self.__recordsToProcess,<EOL>})<EOL>expdir = os.path.join(self.outdir,  "<STR_LIT>")<EOL>self.generateModules(expDesc,  expdir)<EOL>self.descriptions["<STR_LIT>"]=(expdir, expDesc)<EOL>return expdir<EOL>


def runNetwork(network, numRecords, writer):

sensorRegion = network.regions[_RECORD_SENSOR]<EOL>l1SpRegion = network.regions[_L1_SPATIAL_POOLER]<EOL>l1TpRegion = network.regions[_L1_TEMPORAL_MEMORY]<EOL>l1Classifier = network.regions[_L1_CLASSIFIER]<EOL>l2SpRegion = network.regions[_L2_SPATIAL_POOLER]<EOL>l2TpRegion = network.regions[_L2_TEMPORAL_MEMORY]<EOL>l2Classifier = network.regions[_L2_CLASSIFIER]<EOL>l1PreviousPredictedColumns = []<EOL>l2PreviousPredictedColumns = []<EOL>l1PreviousPrediction = None<EOL>l2PreviousPrediction = None<EOL>l1ErrorSum = <NUM_LIT:0.0><EOL>l2ErrorSum = <NUM_LIT:0.0><EOL>for record in range(numRecords):<EOL><INDENT>network.run(<NUM_LIT:1>)<EOL>actual = float(sensorRegion.getOutputData("<STR_LIT>")[<NUM_LIT:0>])<EOL>l1Predictions = l1Classifier.getOutputData("<STR_LIT>")<EOL>l1Probabilities = l1Classifier.getOutputData("<STR_LIT>")<EOL>l1Prediction = l1Predictions[l1Probabilities.argmax()]<EOL>if l1PreviousPrediction is not None:<EOL><INDENT>l1ErrorSum += math.fabs(l1PreviousPrediction - actual)<EOL><DEDENT>l1PreviousPrediction = l1Prediction<EOL>l2Predictions = l2Classifier.getOutputData("<STR_LIT>")<EOL>l2Probabilities = l2Classifier.getOutputData("<STR_LIT>")<EOL>l2Prediction = l2Predictions[l2Probabilities.argmax()]<EOL>if l2PreviousPrediction is not None:<EOL><INDENT>l2ErrorSum += math.fabs(l2PreviousPrediction - actual)<EOL><DEDENT>l2PreviousPrediction = l2Prediction<EOL>l1AnomalyScore = l1TpRegion.getOutputData("<STR_LIT>")[<NUM_LIT:0>]<EOL>l2AnomalyScore = l2TpRegion.getOutputData("<STR_LIT>")[<NUM_LIT:0>]<EOL>writer.writerow((record, actual, l1PreviousPrediction, l1AnomalyScore, l2PreviousPrediction, l2AnomalyScore))<EOL>l1PredictedColumns = l1TpRegion.getOutputData("<STR_LIT>").nonzero()[<NUM_LIT:0>]<EOL>l1PreviousPredictedColumns = copy.deepcopy(l1PredictedColumns)<EOL>l2PredictedColumns = l2TpRegion.getOutputData("<STR_LIT>").nonzero()[<NUM_LIT:0>]<EOL>l2PreviousPredictedColumns = copy.deepcopy(l2PredictedColumns)<EOL><DEDENT>if numRecords > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>" % (l1ErrorSum / (numRecords - <NUM_LIT:1>)))<EOL>print("<STR_LIT>" % (l2ErrorSum / (numRecords - <NUM_LIT:1>)))<EOL><DEDENT>

Runs specified Network writing the ensuing anomaly
scores to writer.

@param network: The Network instance to be run
@param writer: A csv.writer used to write to output file.


network = Network()<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>",<EOL>json.dumps({"<STR_LIT>": _VERBOSITY}))<EOL>sensor = network.regions["<STR_LIT>"].getSelf()<EOL>sensor.encoder = createEncoder()<EOL>sensor.dataSource = dataSource<EOL>SP_PARAMS["<STR_LIT>"] = sensor.encoder.getWidth()<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>", json.dumps(SP_PARAMS))<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>",<EOL>json.dumps(TM_PARAMS))<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>",<EOL>json.dumps({}))<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>spatialPoolerRegion = network.regions["<STR_LIT>"]<EOL>spatialPoolerRegion.setParameter("<STR_LIT>", True)<EOL>spatialPoolerRegion.setParameter("<STR_LIT>", False)<EOL>temporalPoolerRegion = network.regions["<STR_LIT>"]<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>return network<EOL>

Create the Network instance.

    The network has a sensor region reading data from `dataSource` and passing
    the encoded representation to an SPRegion. The SPRegion output is passed to
    a TMRegion.

    :param dataSource: a RecordStream instance to get data from
    :returns: a Network instance ready to run


inputFilePath = recordParams["<STR_LIT>"]<EOL>scalarEncoderArgs = recordParams["<STR_LIT>"]<EOL>dateEncoderArgs = recordParams["<STR_LIT>"]<EOL>scalarEncoder = ScalarEncoder(**scalarEncoderArgs)<EOL>dateEncoder = DateEncoder(**dateEncoderArgs)<EOL>encoder = MultiEncoder()<EOL>encoder.addEncoder(scalarEncoderArgs["<STR_LIT:name>"], scalarEncoder)<EOL>encoder.addEncoder(dateEncoderArgs["<STR_LIT:name>"], dateEncoder)<EOL>network = Network()<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>",<EOL>json.dumps({"<STR_LIT>": verbosity}))<EOL>sensor = network.regions["<STR_LIT>"].getSelf()<EOL>sensor.encoder = encoder<EOL>sensor.dataSource = FileRecordStream(streamID=inputFilePath)<EOL>spatialParams["<STR_LIT>"] = sensor.encoder.getWidth()<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>",<EOL>json.dumps(spatialParams))<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>network.addRegion("<STR_LIT>", "<STR_LIT>",<EOL>json.dumps(temporalParams))<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>network.link("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>srcOutput="<STR_LIT>", destInput="<STR_LIT>")<EOL>spatialPoolerRegion = network.regions["<STR_LIT>"]<EOL>spatialPoolerRegion.setParameter("<STR_LIT>", True)<EOL>spatialPoolerRegion.setParameter("<STR_LIT>", False)<EOL>temporalPoolerRegion = network.regions["<STR_LIT>"]<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>temporalPoolerRegion.setParameter("<STR_LIT>", True)<EOL>return network<EOL>

Generates a Network with connected RecordSensor, SP, TM.

    This function takes care of generating regions and the canonical links.
    The network has a sensor region reading data from a specified input and
    passing the encoded representation to an SPRegion.
    The SPRegion output is passed to a TMRegion.

    Note: this function returns a network that needs to be initialized. This
    allows the user to extend the network by adding further regions and
    connections.

    :param recordParams: a dict with parameters for creating RecordSensor region.
    :param spatialParams: a dict with parameters for creating SPRegion.
    :param temporalParams: a dict with parameters for creating TMRegion.
    :param verbosity: an integer representing how chatty the network will be.


accuracy = []<EOL>filename = os.path.join(os.path.dirname(__file__), "<STR_LIT>")<EOL>with zipfile.ZipFile(filename) as archive:<EOL><INDENT>with archive.open("<STR_LIT>") as datafile:<EOL><INDENT>for _ in range(<NUM_LIT:7>):<EOL><INDENT>next(datafile)<EOL><DEDENT>for _ in range(LEARNING_RECORDS):<EOL><INDENT>next(datafile)<EOL><DEDENT>for _ in range(size):<EOL><INDENT>pages = readUserSession(datafile)<EOL>model.resetSequenceStates()<EOL>for i in range(len(pages) - <NUM_LIT:1>):<EOL><INDENT>result = model.run({"<STR_LIT>": pages[i]})<EOL>inferences = result.inferences["<STR_LIT>"][<NUM_LIT:1>]<EOL>predicted = sorted(list(inferences.items()), key=itemgetter(<NUM_LIT:1>), reverse=True)[:top]<EOL>accuracy.append(<NUM_LIT:1> if pages[i + <NUM_LIT:1>] in list(zip(*predicted))[<NUM_LIT:0>] else <NUM_LIT:0>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return np.mean(accuracy)<EOL>

Compute prediction accuracy by checking if the next page in the sequence is
within the top N predictions calculated by the model
Args:
  model: HTM model
  size: Sample size
  top: top N predictions to use

Returns: Probability the next page in the sequence is within the top N
         predicted pages


if perm['<STR_LIT>'] != perm['<STR_LIT>']:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

This function can be used to selectively filter out specific permutation
    combinations. It is called for every possible permutation of the variables
    in the permutations dict. It should return True for valid a combination of
    permutation values and False for an invalid one.

    Parameters:
    ---------------------------------------------------------
    perm: dict of one possible combination of name:value
                  pairs chosen from permutations.


def analyzeOverlaps(activeCoincsFile, encodingsFile, dataset):

lines = activeCoincsFile.readlines()<EOL>inputs = encodingsFile.readlines()<EOL>w = len(inputs[<NUM_LIT:0>].split('<STR_LIT:U+0020>'))-<NUM_LIT:1><EOL>patterns = set([])<EOL>encodings = set([])<EOL>coincs = []    <EOL>reUsedCoincs = []<EOL>firstLine = inputs[<NUM_LIT:0>].split('<STR_LIT:U+0020>')<EOL>size = int(firstLine.pop(<NUM_LIT:0>))<EOL>spOutput = np.zeros((len(lines),<NUM_LIT>))<EOL>inputBits = np.zeros((len(lines),w))<EOL>print('<STR_LIT>', size)<EOL>print('<STR_LIT>', len(lines), '<STR_LIT:\n>')<EOL>print('<STR_LIT>', w)<EOL>count = <NUM_LIT:0><EOL>for x in range(len(lines)):<EOL><INDENT>inputSpace = []     <EOL>spBUout = [int(z) for z in lines[x].split('<STR_LIT:U+0020>')]  <EOL>spBUout.pop(<NUM_LIT:0>)         <EOL>temp = set(spBUout)<EOL>spOutput[x]=spBUout<EOL>input = [int(z) for z in inputs[x].split('<STR_LIT:U+0020>')]    <EOL>input.pop(<NUM_LIT:0>)   <EOL>tempInput = set(input)<EOL>inputBits[x]=input<EOL>for m in range(size):<EOL><INDENT>if m in tempInput:<EOL><INDENT>inputSpace.append(m)<EOL><DEDENT>else:<EOL><INDENT>inputSpace.append('<STR_LIT:|>')  <EOL><DEDENT><DEDENT>repeatedBits = tempInput.intersection(encodings)    <EOL>reUsed = temp.intersection(patterns)  <EOL>if len(reUsed)==<NUM_LIT:0>:<EOL><INDENT>coincs.append((count,temp,repeatedBits,inputSpace, tempInput))  <EOL><DEDENT>else:<EOL><INDENT>reUsedCoincs.append((count,temp,repeatedBits,inputSpace, tempInput))<EOL><DEDENT>patterns=patterns.union(temp)   <EOL>encodings = encodings.union(tempInput)<EOL>count +=<NUM_LIT:1><EOL><DEDENT>overlap = {}<EOL>overlapVal = <NUM_LIT:0><EOL>seen = []<EOL>seen = (printOverlaps(coincs, coincs, seen))<EOL>print(len(seen), '<STR_LIT>')<EOL>seen = printOverlaps(reUsedCoincs, coincs, seen)<EOL>Summ=[]<EOL>for z in coincs:<EOL><INDENT>c=<NUM_LIT:0><EOL>for y in reUsedCoincs:<EOL><INDENT>c += len(z[<NUM_LIT:1>].intersection(y[<NUM_LIT:1>]))<EOL><DEDENT>Summ.append(c)<EOL><DEDENT>print('<STR_LIT>', Summ)<EOL>for m in range(<NUM_LIT:3>):<EOL><INDENT>displayLimit = min(<NUM_LIT>, len(spOutput[m*<NUM_LIT:200>:]))<EOL>if displayLimit><NUM_LIT:0>:<EOL><INDENT>drawFile(dataset, np.zeros([len(inputBits[:(m+<NUM_LIT:1>)*displayLimit]),len(inputBits[:(m+<NUM_LIT:1>)*displayLimit])]), inputBits[:(m+<NUM_LIT:1>)*displayLimit], spOutput[:(m+<NUM_LIT:1>)*displayLimit], w, m+<NUM_LIT:1>)<EOL><DEDENT>else: <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT>pyl.show()<EOL>

Mirror Image Visualization: Shows the encoding space juxtaposed against the
    coincidence space. The encoding space is the bottom-up sensory encoding and
    the coincidence space depicts the corresponding activation of coincidences in
    the SP. Hence, the mirror image visualization is a visual depiction of the
    mapping of SP cells to the input representations.

    Note:
    * The files spBUOut and sensorBUOut are assumed to be in the output format
    used for LPF experiment outputs.
    * BU outputs for some sample datasets are provided. Specify the name of the
    dataset as an option while running this script.


PLOT_PRECISION = <NUM_LIT:100><EOL>distribMatrix = np.zeros((PLOT_PRECISION+<NUM_LIT:1>,PLOT_PRECISION+<NUM_LIT:1>))<EOL>outputSize = len(outputs)<EOL>for i in range(<NUM_LIT:0>,outputSize):<EOL><INDENT>for j in range(i+<NUM_LIT:1>,outputSize):<EOL><INDENT>in1 = outputs[i]<EOL>in2 = outputs[j]<EOL>dist = (abs(in1-in2) > <NUM_LIT:0.1>)<EOL>intDist = int(dist.sum()/<NUM_LIT:2>+<NUM_LIT:0.1>)<EOL>orig1 = origData[i]<EOL>orig2 = origData[j]<EOL>origDist = (abs(orig1-orig2) > <NUM_LIT:0.1>)<EOL>intOrigDist = int(origDist.sum()/<NUM_LIT:2>+<NUM_LIT:0.1>)<EOL>if intDist < <NUM_LIT:2> and intOrigDist > <NUM_LIT:10>:<EOL><INDENT>print('<STR_LIT>' % (i, j, intDist))<EOL>print('<STR_LIT>' % intOrigDist)<EOL><DEDENT>x = int(PLOT_PRECISION*intDist/<NUM_LIT>)<EOL>y = int(PLOT_PRECISION*intOrigDist/<NUM_LIT>)<EOL>if distribMatrix[x, y] < <NUM_LIT:0.1>:<EOL><INDENT>distribMatrix[x, y] = <NUM_LIT:3><EOL><DEDENT>else:<EOL><INDENT>if distribMatrix[x, y] < <NUM_LIT:10>:<EOL><INDENT>distribMatrix[x, y] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>distribMatrix[<NUM_LIT:4>, <NUM_LIT:50>] = <NUM_LIT:3><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT>] = <NUM_LIT:4><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT>] = <NUM_LIT:5><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT>] = <NUM_LIT:6><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT>] = <NUM_LIT:7><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT>] = <NUM_LIT:8><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT>] = <NUM_LIT:9><EOL>distribMatrix[<NUM_LIT:4>, <NUM_LIT:64>] = <NUM_LIT:10><EOL>return distribMatrix<EOL>

Generates a table where each cell represent a frequency of pairs
    as described below.
    x coordinate is the % difference between input records (origData list),
    y coordinate is the % difference between corresponding output records.


def generateRandomInput(numRecords, elemSize = <NUM_LIT>, numSet = <NUM_LIT>):

inputs = []<EOL>for _ in range(numRecords):<EOL><INDENT>input = np.zeros(elemSize, dtype=realDType)<EOL>for _ in range(<NUM_LIT:0>,numSet):<EOL><INDENT>ind = np.random.random_integers(<NUM_LIT:0>, elemSize-<NUM_LIT:1>, <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>input[ind] = <NUM_LIT:1><EOL><DEDENT>while abs(input.sum() - numSet) > <NUM_LIT:0.1>:<EOL><INDENT>ind = np.random.random_integers(<NUM_LIT:0>, elemSize-<NUM_LIT:1>, <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>input[ind] = <NUM_LIT:1><EOL><DEDENT>inputs.append(input)<EOL><DEDENT>return inputs<EOL>

Generates a set of input record

    Params:
            numRecords - how many records to generate
            elemSize - the size of each record (num 0s or 1s)
            numSet - how many 1s in each record

    Returns: a list of inputs


importName = "<STR_LIT>" % (<EOL>gymName.replace("<STR_LIT:U+0020>", "<STR_LIT:_>").replace("<STR_LIT:->", "<STR_LIT:_>")<EOL>)<EOL>print("<STR_LIT>" % importName)<EOL>try:<EOL><INDENT>importedModelParams = importlib.import_module(importName).MODEL_PARAMS<EOL><DEDENT>except ImportError:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>% gymName)<EOL><DEDENT>return importedModelParams<EOL>

Given a gym name, assumes a matching model params python module exists within
the model_params directory and attempts to import it.
:param gymName: Gym name, used to guess the model params module name.
:return: OPF Model params dictionary


def _generateCategory(filename="<STR_LIT>", numSequences=<NUM_LIT:2>, elementsPerSeq=<NUM_LIT:1>, <EOL>numRepeats=<NUM_LIT:10>, resets=False):

<EOL>scriptDir = os.path.dirname(__file__)<EOL>pathname = os.path.join(scriptDir, '<STR_LIT>', filename)<EOL>print("<STR_LIT>" % (pathname))<EOL>fields = [('<STR_LIT>', '<STR_LIT:int>', '<STR_LIT:R>'), ('<STR_LIT>', '<STR_LIT:int>', '<STR_LIT:C>'),<EOL>('<STR_LIT>', '<STR_LIT:string>', '<STR_LIT>')]  <EOL>outFile = FileRecordStream(pathname, write=True, fields=fields)<EOL>sequences = []<EOL>for i in range(numSequences):<EOL><INDENT>seq = [x for x in range(i*elementsPerSeq, (i+<NUM_LIT:1>)*elementsPerSeq)]<EOL>sequences.append(seq)<EOL><DEDENT>seqIdxs = []<EOL>for i in range(numRepeats):<EOL><INDENT>seqIdxs += list(range(numSequences))<EOL><DEDENT>random.shuffle(seqIdxs)<EOL>for seqIdx in seqIdxs:<EOL><INDENT>reset = int(resets)<EOL>seq = sequences[seqIdx]<EOL>for x in seq:<EOL><INDENT>outFile.appendRecord([reset, str(seqIdx), str(x)])<EOL>reset = <NUM_LIT:0><EOL><DEDENT><DEDENT>outFile.close()<EOL>

Generate a simple dataset. This contains a bunch of non-overlapping
    sequences.

    Parameters:
    ----------------------------------------------------
    filename:       name of the file to produce, including extension. It will
                    be created in a 'datasets' sub-directory within the
                    directory containing this script.
    numSequences:   how many sequences to generate
    elementsPerSeq: length of each sequence
    numRepeats:     how many times to repeat each sequence in the output
    resets:         if True, turn on reset at start of each sequence


def _generateSimple(filename="<STR_LIT>", numSequences=<NUM_LIT:1>, elementsPerSeq=<NUM_LIT:3>, <EOL>numRepeats=<NUM_LIT:10>):

<EOL>scriptDir = os.path.dirname(__file__)<EOL>pathname = os.path.join(scriptDir, '<STR_LIT>', filename)<EOL>print("<STR_LIT>" % (pathname))<EOL>fields = [('<STR_LIT>', '<STR_LIT>', '<STR_LIT:T>'), <EOL>('<STR_LIT>', '<STR_LIT:string>', '<STR_LIT>'),  <EOL>('<STR_LIT>', '<STR_LIT:float>', '<STR_LIT>')]  <EOL>outFile = FileRecordStream(pathname, write=True, fields=fields)<EOL>sequences = []<EOL>for i in range(numSequences):<EOL><INDENT>seq = [x for x in range(i*elementsPerSeq, (i+<NUM_LIT:1>)*elementsPerSeq)]<EOL>sequences.append(seq)<EOL><DEDENT>seqIdxs = []<EOL>for i in range(numRepeats):<EOL><INDENT>seqIdxs += list(range(numSequences))<EOL><DEDENT>random.shuffle(seqIdxs)<EOL>timestamp = datetime.datetime(year=<NUM_LIT>, month=<NUM_LIT:1>, day=<NUM_LIT:1>, hour=<NUM_LIT:0>, minute=<NUM_LIT:0>,<EOL>second=<NUM_LIT:0>)<EOL>timeDelta = datetime.timedelta(hours=<NUM_LIT:1>)<EOL>for seqIdx in seqIdxs:<EOL><INDENT>seq = sequences[seqIdx]<EOL>for x in seq:<EOL><INDENT>outFile.appendRecord([timestamp, str(x), x])<EOL>timestamp += timeDelta<EOL><DEDENT><DEDENT>for seqIdx in seqIdxs:<EOL><INDENT>seq = sequences[seqIdx]<EOL>for i,x in enumerate(seq):<EOL><INDENT>if i != <NUM_LIT:1>:<EOL><INDENT>outFile.appendRecord([timestamp, str(x), x])<EOL><DEDENT>timestamp += timeDelta<EOL><DEDENT><DEDENT>for seqIdx in seqIdxs:<EOL><INDENT>seq = sequences[seqIdx]<EOL>for i,x in enumerate(seq):<EOL><INDENT>if i != <NUM_LIT:1>:<EOL><INDENT>outFile.appendRecord([timestamp, str(x), x])<EOL><DEDENT>timestamp += timeDelta<EOL><DEDENT><DEDENT>for seqIdx in seqIdxs:<EOL><INDENT>seq = sequences[seqIdx]<EOL>for x in seq:<EOL><INDENT>outFile.appendRecord([timestamp, str(x), x])<EOL>timestamp += timeDelta<EOL><DEDENT><DEDENT>outFile.close()<EOL>

Generate a simple dataset. This contains a bunch of non-overlapping
    sequences.

    At the end of the dataset, we introduce missing records so that test
    code can insure that the model didn't get confused by them.

    Parameters:
    ----------------------------------------------------
    filename:       name of the file to produce, including extension. It will
                    be created in a 'datasets' sub-directory within the
                    directory containing this script.
    numSequences:   how many sequences to generate
    elementsPerSeq: length of each sequence
    numRepeats:     how many times to repeat each sequence in the output


<EOL>return True<EOL>

This function can be used to selectively filter out specific permutation
    combinations. It is called by RunPermutations for every possible permutation
    of the variables in the permutations dict. It should return True for valid a
    combination of permutation values and False for an invalid one.

    Parameters:
    ---------------------------------------------------------
    perm: dict of one possible combination of name:value
          pairs chosen from permutations.


<EOL>numCategories = <NUM_LIT:5><EOL>initProb = numpy.zeros(numCategories)<EOL>initProb[<NUM_LIT:0>] = <NUM_LIT:1.0><EOL>firstOrder = dict()<EOL>firstOrder['<STR_LIT:0>'] = numpy.array([<NUM_LIT:0>, <NUM_LIT:0.1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT>])<EOL>firstOrder['<STR_LIT:1>'] = numpy.array([<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT:0>])<EOL>firstOrder['<STR_LIT:2>'] = numpy.array([<NUM_LIT:1.0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>])<EOL>firstOrder['<STR_LIT:3>'] = numpy.array([<NUM_LIT:1.0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>])<EOL>firstOrder['<STR_LIT:4>'] = numpy.array([<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0.5>, <NUM_LIT:0.5>, <NUM_LIT:0>])<EOL>secondOrder = None<EOL>categoryList = ['<STR_LIT>' % x for x in range(<NUM_LIT:5>)]<EOL>return (initProb, firstOrder, secondOrder, <NUM_LIT:3>, categoryList)<EOL>

Generate the initial, first order, and second order transition
    probabilities for 'probability0'. For this model, we generate the following
    set of sequences:

      .1   .75
    0----1-----2
     \    \
      \    \  .25
       \    \-----3
        \
         \ .9     .5
          \--- 4--------- 2
                \
                 \   .5
                  \---------3

    Parameters:
    ----------------------------------------------------------------------
    retval: (initProb, firstOrder, secondOrder, seqLen)
              initProb:     Initial probability for each category. This is a vector
                              of length len(categoryList).
              firstOrder:   A dictionary of the 1st order probabilities. The key
                              is the 1st element of the sequence, the value is
                              the probability of each 2nd element given the first.
              secondOrder:  A dictionary of the 2nd order probabilities. The key
                              is the first 2 elements of the sequence, the value is
                              the probability of each possible 3rd element given the
                              first two.
              seqLen:       Desired length of each sequence. The 1st element will
                            be generated using the initProb, the 2nd element by the
                            firstOrder table, and the 3rd and all successive
                            elements by the secondOrder table.
              categoryList:  list of category names to use

    Here is an example of some return values when there are 3 categories
    initProb:         [0.7, 0.2, 0.1]

    firstOrder:       {'[0]': [0.3, 0.3, 0.4],
                       '[1]': [0.3, 0.3, 0.4],
                       '[2]': [0.3, 0.3, 0.4]}

    secondOrder:      {'[0,0]': [0.3, 0.3, 0.4],
                       '[0,1]': [0.3, 0.3, 0.4],
                       '[0,2]': [0.3, 0.3, 0.4],
                       '[1,0]': [0.3, 0.3, 0.4],
                       '[1,1]': [0.3, 0.3, 0.4],
                       '[1,2]': [0.3, 0.3, 0.4],
                       '[2,0]': [0.3, 0.3, 0.4],
                       '[2,1]': [0.3, 0.3, 0.4],
                       '[2,2]': [0.3, 0.3, 0.4]}


def _generateFileFromProb(filename, numRecords, categoryList, initProb, <EOL>firstOrderProb, secondOrderProb, seqLen, numNoise=<NUM_LIT:0>, resetsEvery=None):

<EOL>print("<STR_LIT>" % (filename))<EOL>fields = [('<STR_LIT>', '<STR_LIT:int>', '<STR_LIT:R>'), <EOL>('<STR_LIT>', '<STR_LIT:string>', '<STR_LIT>'),<EOL>('<STR_LIT>', '<STR_LIT:float>', '<STR_LIT>')]<EOL>scriptDir = os.path.dirname(__file__)<EOL>pathname = os.path.join(scriptDir, '<STR_LIT>', filename)<EOL>outFile = FileRecordStream(pathname, write=True, fields=fields)<EOL>initCumProb = initProb.cumsum()<EOL>firstOrderCumProb = dict()<EOL>for (key,value) in firstOrderProb.items():<EOL><INDENT>firstOrderCumProb[key] = value.cumsum()<EOL><DEDENT>if secondOrderProb is not None:<EOL><INDENT>secondOrderCumProb = dict()<EOL>for (key,value) in secondOrderProb.items():<EOL><INDENT>secondOrderCumProb[key] = value.cumsum()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>secondOrderCumProb = None<EOL><DEDENT>elementsInSeq = []<EOL>numElementsSinceReset = <NUM_LIT:0><EOL>maxCatIdx = len(categoryList) - <NUM_LIT:1><EOL>for _ in range(numRecords):<EOL><INDENT>if numElementsSinceReset == <NUM_LIT:0>:<EOL><INDENT>reset = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>reset = <NUM_LIT:0><EOL><DEDENT>rand = numpy.random.rand()<EOL>if secondOrderCumProb is None:<EOL><INDENT>if len(elementsInSeq) == <NUM_LIT:0>:<EOL><INDENT>catIdx = numpy.searchsorted(initCumProb, rand)<EOL><DEDENT>elif len(elementsInSeq) >= <NUM_LIT:1> and(seqLen is None or len(elementsInSeq) < seqLen-numNoise):<EOL><INDENT>catIdx = numpy.searchsorted(firstOrderCumProb[str(elementsInSeq[-<NUM_LIT:1>])], <EOL>rand)<EOL><DEDENT>else:   <EOL><INDENT>catIdx = numpy.random.randint(len(categoryList))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(elementsInSeq) == <NUM_LIT:0>:<EOL><INDENT>catIdx = numpy.searchsorted(initCumProb, rand)<EOL><DEDENT>elif len(elementsInSeq) == <NUM_LIT:1>:<EOL><INDENT>catIdx = numpy.searchsorted(firstOrderCumProb[str(elementsInSeq)], rand)<EOL><DEDENT>elif (len(elementsInSeq) >=<NUM_LIT:2>) and(seqLen is None or len(elementsInSeq) < seqLen-numNoise):<EOL><INDENT>catIdx = numpy.searchsorted(secondOrderCumProb[str(elementsInSeq[-<NUM_LIT:2>:])], rand)<EOL><DEDENT>else:   <EOL><INDENT>catIdx = numpy.random.randint(len(categoryList))<EOL><DEDENT><DEDENT>catIdx = min(maxCatIdx, catIdx)<EOL>outFile.appendRecord([reset, categoryList[catIdx], catIdx])    <EOL>elementsInSeq.append(catIdx)<EOL>numElementsSinceReset += <NUM_LIT:1><EOL>if resetsEvery is not None and numElementsSinceReset == resetsEvery:<EOL><INDENT>numElementsSinceReset = <NUM_LIT:0><EOL>elementsInSeq = []<EOL><DEDENT>if seqLen is not None and (len(elementsInSeq) == seqLen+numNoise):<EOL><INDENT>elementsInSeq = []<EOL><DEDENT><DEDENT>outFile.close()<EOL>

Generate a set of records reflecting a set of probabilities.

    Parameters:
    ----------------------------------------------------------------
    filename:         name of .csv file to generate
    numRecords:       number of records to generate
    categoryList:     list of category names
    initProb:         Initial probability for each category. This is a vector
                        of length len(categoryList).
    firstOrderProb:   A dictionary of the 1st order probabilities. The key
                        is the 1st element of the sequence, the value is
                        the probability of each 2nd element given the first.
    secondOrderProb:  A dictionary of the 2nd order probabilities. The key
                        is the first 2 elements of the sequence, the value is
                        the probability of each possible 3rd element given the
                        first two. If this is None, then the sequences will be
                        first order only.
    seqLen:           Desired length of each sequence. The 1st element will
                        be generated using the initProb, the 2nd element by the
                        firstOrder table, and the 3rd and all successive
                        elements by the secondOrder table. None means infinite
                        length.
    numNoise:         Number of noise elements to place between each
                        sequence. The noise elements are evenly distributed from
                        all categories.
    resetsEvery:      If not None, generate a reset every N records

    Here is an example of some parameters:

    categoryList:     ['cat1', 'cat2', 'cat3']

    initProb:         [0.7, 0.2, 0.1]

    firstOrderProb:   {'[0]': [0.3, 0.3, 0.4],
                       '[1]': [0.3, 0.3, 0.4],
                       '[2]': [0.3, 0.3, 0.4]}

    secondOrderProb:  {'[0,0]': [0.3, 0.3, 0.4],
                       '[0,1]': [0.3, 0.3, 0.4],
                       '[0,2]': [0.3, 0.3, 0.4],
                       '[1,0]': [0.3, 0.3, 0.4],
                       '[1,1]': [0.3, 0.3, 0.4],
                       '[1,2]': [0.3, 0.3, 0.4],
                       '[2,0]': [0.3, 0.3, 0.4],
                       '[2,1]': [0.3, 0.3, 0.4],
                       '[2,2]': [0.3, 0.3, 0.4]}


newn = int(<NUM_LIT:1.0> * len(l) / n + <NUM_LIT:0.5>)<EOL>for i in range(<NUM_LIT:0>, n-<NUM_LIT:1>):<EOL><INDENT>yield l[i*newn:i*newn+newn]<EOL><DEDENT>yield l[n*newn-newn:]<EOL>


def slice_sampler(px, N = <NUM_LIT:1>, x = None):

values = np.zeros(N, dtype=np.int)<EOL>samples = np.arange(len(px))<EOL>px = np.array(px) / (<NUM_LIT:1.>*sum(px))<EOL>u = uniform(<NUM_LIT:0>, max(px))<EOL>for n in range(N):<EOL><INDENT>included = px>=u<EOL>choice = random.sample(list(range(np.sum(included))), <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>values[n] = samples[included][choice]<EOL>u = uniform(<NUM_LIT:0>, px[included][choice])<EOL><DEDENT>if x:<EOL><INDENT>if len(x) == len(px):<EOL><INDENT>x=np.array(x)<EOL>values = x[values]<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>return values<EOL>

Provides samples from a user-defined distribution.

slice_sampler(px, N = 1, x = None)

Inputs:
px = A discrete probability distribution.
N  = Number of samples to return, default is 1
x  = Optional list/array of observation values to return, where prob(x) = px.

Outputs:
If x=None (default) or if len(x) != len(px), it will return an array of integers
between 0 and len(px)-1. If x is supplied, it will return the
samples from x according to the distribution px.


size = len(x1)<EOL>for i in range(size):<EOL><INDENT>x2[i] = x1[i]<EOL><DEDENT>

Copies the contents of vector x1 into vector x2.

@param x1 (array) binary vector to be copied
@param x2 (array) binary vector where x1 is copied


nonZeroX1 = np.count_nonzero(x1)<EOL>nonZeroX2 = np.count_nonzero(x2)<EOL>minX1X2 = min(nonZeroX1, nonZeroX2)<EOL>percentOverlap = <NUM_LIT:0><EOL>if minX1X2 > <NUM_LIT:0>:<EOL><INDENT>percentOverlap = float(np.dot(x1, x2))/float(minX1X2)<EOL><DEDENT>return percentOverlap<EOL>

Computes the percentage of overlap between vectors x1 and x2.

@param x1   (array) binary vector
@param x2   (array) binary vector
@param size (int)   length of binary vectors

@return percentOverlap (float) percentage overlap between x1 and x2


def buildTrainingSet(numSequences = <NUM_LIT:2>,<EOL>sequenceLength = <NUM_LIT:100>,<EOL>pctShared = <NUM_LIT>,<EOL>seqGenMode = '<STR_LIT>',<EOL>subsequenceStartPos = <NUM_LIT:10>,<EOL>numCols = <NUM_LIT:100>,<EOL>minOnes=<NUM_LIT>,<EOL>maxOnes = <NUM_LIT>,<EOL>disjointConsecutive =True):

<EOL>colSet = set(range(numCols))<EOL>if '<STR_LIT>' in seqGenMode:<EOL><INDENT>assert '<STR_LIT>' in seqGenMode and '<STR_LIT>' not in seqGenMode<EOL><DEDENT>if '<STR_LIT>' in seqGenMode or numSequences == <NUM_LIT:1>:<EOL><INDENT>pctShared = <NUM_LIT:0.0><EOL><DEDENT>if '<STR_LIT>' not in seqGenMode and '<STR_LIT>' not in seqGenMode:<EOL><INDENT>sharedSequenceLength = int(pctShared*sequenceLength)<EOL><DEDENT>elif '<STR_LIT>' in seqGenMode:<EOL><INDENT>sharedSequenceLength = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>sharedSequenceLength = <NUM_LIT:0><EOL><DEDENT>assert sharedSequenceLength + subsequenceStartPos < sequenceLength<EOL>sharedSequence = []<EOL>for i in range(sharedSequenceLength):<EOL><INDENT>if disjointConsecutive and i > <NUM_LIT:0>:<EOL><INDENT>x = generatePattern(numCols, minOnes, maxOnes, colSet,<EOL>sharedSequence[i-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>x = generatePattern(numCols, minOnes, maxOnes, colSet)<EOL><DEDENT>sharedSequence.append(x)<EOL><DEDENT>trainingSequences = []<EOL>if '<STR_LIT>' not in seqGenMode:<EOL><INDENT>trailingLength = sequenceLength - sharedSequenceLength - subsequenceStartPos<EOL><DEDENT>else:<EOL><INDENT>trailingLength = sequenceLength - sharedSequenceLength<EOL><DEDENT>for k,s in enumerate(range(numSequences)):<EOL><INDENT>if len(trainingSequences) > <NUM_LIT:0> and '<STR_LIT>' in seqGenMode:<EOL><INDENT>r = list(range(subsequenceStartPos))+ list(range(subsequenceStartPos + sharedSequenceLength, sequenceLength))<EOL>rgen.shuffle(r)<EOL>r = r[:subsequenceStartPos]+ list(range(subsequenceStartPos, subsequenceStartPos + sharedSequenceLength))+ r[subsequenceStartPos:]<EOL>sequence = [trainingSequences[k-<NUM_LIT:1>][j] for j in r]<EOL><DEDENT>else:<EOL><INDENT>sequence = []<EOL>if '<STR_LIT>' not in seqGenMode:<EOL><INDENT>for i in range(subsequenceStartPos):<EOL><INDENT>if disjointConsecutive and i > <NUM_LIT:0>:<EOL><INDENT>x = generatePattern(numCols, minOnes, maxOnes, colSet, sequence[i-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>x = generatePattern(numCols, minOnes, maxOnes, colSet)<EOL><DEDENT>sequence.append(x)<EOL><DEDENT><DEDENT>if '<STR_LIT>' in seqGenMode and '<STR_LIT>' not in seqGenMode:<EOL><INDENT>sequence.extend(sharedSequence)<EOL><DEDENT>for i in range(trailingLength):<EOL><INDENT>if disjointConsecutive and i > <NUM_LIT:0>:<EOL><INDENT>x = generatePattern(numCols, minOnes, maxOnes, colSet, sequence[i-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>x = generatePattern(numCols, minOnes, maxOnes, colSet)<EOL><DEDENT>sequence.append(x)<EOL><DEDENT><DEDENT>assert len(sequence) == sequenceLength<EOL>trainingSequences.append(sequence)<EOL><DEDENT>assert len(trainingSequences) == numSequences<EOL>if VERBOSITY >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>")<EOL>pprint.pprint(trainingSequences)<EOL><DEDENT>if sharedSequenceLength > <NUM_LIT:0>:<EOL><INDENT>return (trainingSequences, subsequenceStartPos + sharedSequenceLength)<EOL><DEDENT>else:<EOL><INDENT>return (trainingSequences, -<NUM_LIT:1>)<EOL><DEDENT>

Build random high order test sequences.

    Parameters:
    --------------------------------------------
    numSequences:           The number of sequences created.
    sequenceLength:         The length of each sequence.
    pctShared:              The percentage of sequenceLength that is shared across
                            every sequence. If sequenceLength is 100 and pctShared
                            is 0.2, then a subsequence consisting of 20 patterns
                            will be in every sequence. Can also be the keyword
                            'one pattern', in which case a single time step is
                            shared.
    seqGenMode:             What kind of sequence to generate. If contains 'shared'
                            generates shared subsequence. If contains 'no shared',
                            does not generate any shared subsequence. If contains
                            'shuffle', will use common patterns shuffle among the
                            different sequences. If contains 'beginning', will
                            place shared subsequence at the beginning.
    subsequenceStartPos:    The position where the shared subsequence starts
    numCols:                Number of columns in each pattern.
    minOnes:                The minimum number of 1's in each pattern.
    maxOnes:                The maximum number of 1's in each pattern.
    disjointConsecutive:    Whether to generate disjoint consecutive patterns or not.


numPatterns = <NUM_LIT><EOL>p = getSimplePatterns(numOnes, numPatterns)<EOL>s = []<EOL>s.append(p[rgen.randint(<NUM_LIT:5>,numPatterns)])<EOL>for _ in range(<NUM_LIT:50>):<EOL><INDENT>r = rgen.randint(<NUM_LIT:5>,numPatterns)<EOL>print(r, end='<STR_LIT:U+0020>')<EOL>s.append(p[r])<EOL>if rgen.binomial(<NUM_LIT:1>, <NUM_LIT:0.5>) > <NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>s.append(p[<NUM_LIT:0>])<EOL>s.append(p[<NUM_LIT:1>])<EOL>s.append(p[<NUM_LIT:2>])<EOL>s.append(p[<NUM_LIT:4>])<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>s.append(p[<NUM_LIT:1>])<EOL>s.append(p[<NUM_LIT:2>])<EOL>s.append(p[<NUM_LIT:3>])<EOL><DEDENT>r = rgen.randint(<NUM_LIT:5>,numPatterns)<EOL>s.append(p[r])<EOL>print(r, end='<STR_LIT:U+0020>')<EOL><DEDENT>print()<EOL>return ([s], [ [p[<NUM_LIT:0>], p[<NUM_LIT:1>], p[<NUM_LIT:2>], p[<NUM_LIT:4>]],  [p[<NUM_LIT:1>], p[<NUM_LIT:2>], p[<NUM_LIT:3>]] ])<EOL>

Simple sequences for HL0b. Each pattern in the sequence has a series of 1's
    in a specific set of columns.
      There are 23 patterns, p0 to p22.
      The sequences we want to learn are p1->p2->p3 and p0->p1->p2->p4.
      We create a very long sequence consisting of these two sub-sequences
      intermixed with noise, such as:
            N N p0 p1 p2 p4 N N p1 p2 p3 N N p1 p2 p3
      N is randomly chosen from p5 to p22


def TestH2a(sequenceLength, nTests, cellsPerColumn, numCols =<NUM_LIT:100>, nSequences =[<NUM_LIT:2>],<EOL>pctShared = <NUM_LIT>, seqGenMode = '<STR_LIT>',<EOL>shouldFail = False):

print("<STR_LIT>")<EOL>nFailed = <NUM_LIT:0><EOL>subsequenceStartPos = <NUM_LIT:10><EOL>assert subsequenceStartPos < sequenceLength<EOL>for numSequences in nSequences:<EOL><INDENT>print("<STR_LIT>",sequenceLength, end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>",cellsPerColumn,"<STR_LIT>",nTests,"<STR_LIT>", numCols)<EOL>print("<STR_LIT>",numSequences, "<STR_LIT>", pctShared, end='<STR_LIT:U+0020>')<EOL>print("<STR_LIT>", seqGenMode)<EOL>for _ in range(nTests): <EOL><INDENT>trainingSet = buildTrainingSet(numSequences = numSequences,<EOL>sequenceLength = sequenceLength,<EOL>pctShared = pctShared, seqGenMode = seqGenMode,<EOL>subsequenceStartPos = subsequenceStartPos,<EOL>numCols = numCols,<EOL>minOnes = <NUM_LIT>, maxOnes = <NUM_LIT>)<EOL>print("<STR_LIT>")<EOL>numFailures3, numStrictErrors3, numPerfect3, tm3 =testSequence(trainingSet,<EOL>nTrainingReps = <NUM_LIT:10>,<EOL>numberOfCols = numCols,<EOL>cellsPerColumn = cellsPerColumn,<EOL>initialPerm = <NUM_LIT>,<EOL>connectedPerm = <NUM_LIT>,<EOL>minThreshold = <NUM_LIT:12>,<EOL>permanenceInc = <NUM_LIT>,<EOL>permanenceDec = <NUM_LIT:0.1>,<EOL>permanenceMax = <NUM_LIT:1>,<EOL>globalDecay = <NUM_LIT>,<EOL>newSynapseCount = <NUM_LIT:15>,<EOL>activationThreshold = <NUM_LIT:12>,<EOL>doPooling = False,<EOL>shouldFail = shouldFail)<EOL>print("<STR_LIT>")<EOL>numFailures, numStrictErrors, numPerfect, tm2 =testSequence(trainingSet,<EOL>nTrainingReps = <NUM_LIT:2>,<EOL>numberOfCols = numCols,<EOL>cellsPerColumn = cellsPerColumn,<EOL>initialPerm = <NUM_LIT>,<EOL>connectedPerm = <NUM_LIT>,<EOL>minThreshold = <NUM_LIT:12>,<EOL>permanenceInc = <NUM_LIT>,<EOL>permanenceDec = <NUM_LIT:0>,<EOL>permanenceMax = <NUM_LIT:1>,<EOL>globalDecay = <NUM_LIT>,<EOL>newSynapseCount = <NUM_LIT:15>,<EOL>activationThreshold = <NUM_LIT:12>,<EOL>doPooling = False,<EOL>shouldFail = shouldFail)<EOL>print("<STR_LIT>")<EOL>numFailures1, numStrictErrors1, numPerfect1, tm1 =testSequence(trainingSet,<EOL>nTrainingReps = <NUM_LIT:1>,<EOL>numberOfCols = numCols,<EOL>cellsPerColumn = cellsPerColumn,<EOL>initialPerm = <NUM_LIT>,<EOL>connectedPerm = <NUM_LIT>,<EOL>minThreshold = <NUM_LIT:12>,<EOL>permanenceInc = <NUM_LIT>,<EOL>permanenceDec = <NUM_LIT:0>,<EOL>permanenceMax = <NUM_LIT:1>,<EOL>globalDecay = <NUM_LIT>,<EOL>newSynapseCount = <NUM_LIT:15>,<EOL>activationThreshold = <NUM_LIT:12>,<EOL>doPooling = False,<EOL>shouldFail = shouldFail)<EOL>segmentInfo1 = tm1.getSegmentInfo()<EOL>segmentInfo2 = tm2.getSegmentInfo()<EOL>if (abs(segmentInfo1[<NUM_LIT:0>] - segmentInfo2[<NUM_LIT:0>]) > <NUM_LIT:3>) or(abs(segmentInfo1[<NUM_LIT:1>] - segmentInfo2[<NUM_LIT:1>]) > <NUM_LIT:3>*<NUM_LIT:15>) :<EOL><INDENT>print("<STR_LIT>")<EOL>print(segmentInfo1)<EOL>print(segmentInfo2)<EOL>print(tm3.getSegmentInfo())<EOL>tm3.trimSegments()<EOL>print(tm3.getSegmentInfo())<EOL>print("<STR_LIT>")<EOL>print(numFailures1, numStrictErrors1, numPerfect1)<EOL>print(numFailures, numStrictErrors, numPerfect)<EOL>print(numFailures3, numStrictErrors3, numPerfect3)<EOL>numFailures += <NUM_LIT:1><EOL><DEDENT>if numFailures == <NUM_LIT:0> and not shouldFailor numFailures > <NUM_LIT:0> and shouldFail:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL>if shouldFail:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>nFailed = nFailed + <NUM_LIT:1><EOL>print("<STR_LIT>", numFailures)<EOL>print("<STR_LIT>", numStrictErrors)<EOL>print("<STR_LIT>", numPerfect)<EOL><DEDENT><DEDENT><DEDENT>return nFailed<EOL>

Still need to test:
    Two overlapping sequences. OK to get new segments but check that we can
    get correct high order prediction after multiple reps.


numPatterns = <NUM_LIT><EOL>p = getSimplePatterns(numOnes, numPatterns)<EOL>s = []<EOL>s.append(p[rgen.randint(<NUM_LIT:3>,<NUM_LIT>)])<EOL>for _ in range(<NUM_LIT:20>):<EOL><INDENT>s.append(p[rgen.randint(<NUM_LIT:3>,<NUM_LIT>)])<EOL>s.append(p[<NUM_LIT:0>])<EOL>s.append(p[<NUM_LIT:1>])<EOL>s.append(p[<NUM_LIT:2>])<EOL>s.append(p[rgen.randint(<NUM_LIT:3>,<NUM_LIT>)])<EOL><DEDENT>return ([s], [[p[<NUM_LIT:0>], p[<NUM_LIT:1>], p[<NUM_LIT:2>]]])<EOL>

Simple sequences for HL0. Each pattern in the sequence has a series of 1's
    in a specific set of columns.
      There are 23 patterns, p0 to p22.
      The sequence we want to learn is p0->p1->p2
      We create a very long sequence consisting of N N p0 p1 p2 N N p0 p1 p2
      N is randomly chosen from p3 to p22


def findAcceptablePatterns(tm, t, whichSequence, trainingSequences, nAcceptable = <NUM_LIT:1>):

<EOL>upTo = t + <NUM_LIT:2> <EOL>if tm.doPooling:<EOL><INDENT>upTo += min(tm.segUpdateValidDuration, nAcceptable)<EOL><DEDENT>assert upTo <= len(trainingSequences[whichSequence])<EOL>acceptablePatterns = []<EOL>if len(trainingSequences) == <NUM_LIT:2> and(trainingSequences[<NUM_LIT:0>][<NUM_LIT:0>] == trainingSequences[<NUM_LIT:1>][<NUM_LIT:0>]).all():<EOL><INDENT>if (trainingSequences[<NUM_LIT:0>][t] == trainingSequences[<NUM_LIT:1>][t]).all()and (trainingSequences[<NUM_LIT:0>][t+<NUM_LIT:1>] != trainingSequences[<NUM_LIT:1>][t+<NUM_LIT:1>]).any():<EOL><INDENT>acceptablePatterns.append(trainingSequences[<NUM_LIT:0>][t+<NUM_LIT:1>])<EOL>acceptablePatterns.append(trainingSequences[<NUM_LIT:1>][t+<NUM_LIT:1>])<EOL><DEDENT><DEDENT>acceptablePatterns += [trainingSequences[whichSequence][t]for t in range(t,upTo)]<EOL>return acceptablePatterns<EOL>

Tries to infer the set of acceptable patterns for prediction at the given
time step and for the give sequence. Acceptable patterns are: the current one,
plus a certain number of patterns after timeStep, in the sequence that the TM
is currently tracking. Any other pattern is not acceptable.

TODO:
====
- Doesn't work for noise cases.
- Might run in trouble if shared subsequence at the beginning.

Parameters:
==========
tm                       the whole TM, so that we can look at its parameters
t                        the current time step
whichSequence            the sequence we are currently tracking
trainingSequences        all the training sequences
nAcceptable              the number of steps forward from the current timeStep
                         we are willing to consider acceptable. In the case of
                         pooling, it is less than or equal to the min of the
                         number of training reps and the segUpdateValidDuration
                         parameter of the TM, depending on the test case.
                         The default value is 1, because by default, the pattern
                         after the current one should always be predictable.

Return value:
============
acceptablePatterns       A list of acceptable patterns for prediction.


def buildSequencePool(numSequences = <NUM_LIT:10>,<EOL>seqLen = [<NUM_LIT:2>,<NUM_LIT:3>,<NUM_LIT:4>],<EOL>numPatterns = <NUM_LIT:5>,<EOL>numOnBitsPerPattern = <NUM_LIT:3>,<EOL>patternOverlap = <NUM_LIT:0>,<EOL>**kwargs<EOL>):

<EOL>patterns = getSimplePatterns(numOnBitsPerPattern, numPatterns, patternOverlap)<EOL>numCols = len(patterns[<NUM_LIT:0>])<EOL>trainingSequences = []<EOL>for _ in range(numSequences):<EOL><INDENT>sequence = []<EOL>length = random.choice(seqLen)<EOL>for _ in range(length):<EOL><INDENT>patIdx = random.choice(range(numPatterns))<EOL>sequence.append(patterns[patIdx])<EOL><DEDENT>trainingSequences.append(sequence)<EOL><DEDENT>if VERBOSITY >= <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>")<EOL>printAllTrainingSequences(trainingSequences)<EOL><DEDENT>return (numCols, trainingSequences)<EOL>

Create a bunch of sequences of various lengths, all built from
    a fixed set of patterns.

    Parameters:
    -----------------------------------------------------
    numSequences:         Number of training sequences to generate
    seqLen:               List of possible sequence lengths
    numPatterns:          How many possible patterns there are to use within
                            sequences
    numOnBitsPerPattern:  Number of ON bits in each TM input pattern
    patternOverlap:       Max number of bits of overlap between any 2 patterns
    retval:               (numCols, trainingSequences)
                            numCols - width of the patterns
                            trainingSequences - a list of training sequences


print('<STR_LIT>'.join('<STR_LIT:1>' if k != <NUM_LIT:0> else '<STR_LIT:.>' for k in x))<EOL>


def getSimplePatterns(numOnes, numPatterns, patternOverlap=<NUM_LIT:0>):

assert (patternOverlap < numOnes)<EOL>numNewBitsInEachPattern = numOnes - patternOverlap<EOL>numCols = numNewBitsInEachPattern * numPatterns + patternOverlap<EOL>p = []<EOL>for i in range(numPatterns):<EOL><INDENT>x = numpy.zeros(numCols, dtype='<STR_LIT>')<EOL>startBit = i*numNewBitsInEachPattern<EOL>nextStartBit = startBit + numOnes<EOL>x[startBit:nextStartBit] = <NUM_LIT:1><EOL>p.append(x)<EOL><DEDENT>return p<EOL>

Very simple patterns. Each pattern has numOnes consecutive
    bits on. The amount of overlap between consecutive patterns is
    configurable, via the patternOverlap parameter.

    Parameters:
    -----------------------------------------------------------------------
    numOnes:        Number of bits ON in each pattern
    numPatterns:    Number of unique patterns to generate
    patternOverlap: Number of bits of overlap between each successive pattern
    retval:         patterns


def createTMs(includeCPP = True,<EOL>includePy = True,<EOL>numCols = <NUM_LIT:100>,<EOL>cellsPerCol = <NUM_LIT:4>,<EOL>activationThreshold = <NUM_LIT:3>,<EOL>minThreshold = <NUM_LIT:3>,<EOL>newSynapseCount = <NUM_LIT:3>,<EOL>initialPerm = <NUM_LIT>,<EOL>permanenceInc = <NUM_LIT:0.1>,<EOL>permanenceDec = <NUM_LIT:0.0>,<EOL>globalDecay = <NUM_LIT:0.0>,<EOL>pamLength = <NUM_LIT:0>,<EOL>checkSynapseConsistency = True,<EOL>maxInfBacktrack = <NUM_LIT:0>,<EOL>maxLrnBacktrack = <NUM_LIT:0>,<EOL>**kwargs<EOL>):

<EOL>connectedPerm = <NUM_LIT:0.5><EOL>tms = dict()<EOL>if includeCPP:<EOL><INDENT>if VERBOSITY >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>cpp_tm = BacktrackingTMCPP(numberOfCols = numCols, cellsPerColumn = cellsPerCol,<EOL>initialPerm = initialPerm, connectedPerm = connectedPerm,<EOL>minThreshold = minThreshold, newSynapseCount = newSynapseCount,<EOL>permanenceInc = permanenceInc, permanenceDec = permanenceDec,<EOL>activationThreshold = activationThreshold,<EOL>globalDecay = globalDecay, burnIn = <NUM_LIT:1>,<EOL>seed=SEED, verbosity=VERBOSITY,<EOL>checkSynapseConsistency = checkSynapseConsistency,<EOL>collectStats = True,<EOL>pamLength = pamLength,<EOL>maxInfBacktrack = maxInfBacktrack,<EOL>maxLrnBacktrack = maxLrnBacktrack,<EOL>)<EOL>cpp_tm.retrieveLearningStates = True<EOL>tms['<STR_LIT>'] = cpp_tm<EOL><DEDENT>if includePy:<EOL><INDENT>if VERBOSITY >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>py_tm = BacktrackingTM(numberOfCols = numCols,<EOL>cellsPerColumn = cellsPerCol,<EOL>initialPerm = initialPerm,<EOL>connectedPerm = connectedPerm,<EOL>minThreshold = minThreshold,<EOL>newSynapseCount = newSynapseCount,<EOL>permanenceInc = permanenceInc,<EOL>permanenceDec = permanenceDec,<EOL>activationThreshold = activationThreshold,<EOL>globalDecay = globalDecay, burnIn = <NUM_LIT:1>,<EOL>seed=SEED, verbosity=VERBOSITY,<EOL>collectStats = True,<EOL>pamLength = pamLength,<EOL>maxInfBacktrack = maxInfBacktrack,<EOL>maxLrnBacktrack = maxLrnBacktrack,<EOL>)<EOL>tms['<STR_LIT>'] = py_tm<EOL><DEDENT>return tms<EOL>

Create one or more TM instances, placing each into a dict keyed by
    name.

    Parameters:
    ------------------------------------------------------------------
    retval:   tms - dict of TM instances


def _createTMs(self, numCols, fixedResources=False,<EOL>checkSynapseConsistency = True):

<EOL>minThreshold = <NUM_LIT:4><EOL>activationThreshold = <NUM_LIT:8><EOL>newSynapseCount = <NUM_LIT:15><EOL>initialPerm = <NUM_LIT><EOL>connectedPerm = <NUM_LIT:0.5><EOL>permanenceInc = <NUM_LIT:0.1><EOL>permanenceDec = <NUM_LIT><EOL>if fixedResources:<EOL><INDENT>permanenceDec = <NUM_LIT:0.1><EOL>maxSegmentsPerCell = <NUM_LIT:5><EOL>maxSynapsesPerSegment = <NUM_LIT:15><EOL>globalDecay = <NUM_LIT:0><EOL>maxAge = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>permanenceDec = <NUM_LIT><EOL>maxSegmentsPerCell = -<NUM_LIT:1><EOL>maxSynapsesPerSegment = -<NUM_LIT:1><EOL>globalDecay = <NUM_LIT><EOL>maxAge = <NUM_LIT:1><EOL><DEDENT>if g_testCPPTM:<EOL><INDENT>if g_options.verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>cppTM = BacktrackingTMCPP(numberOfCols = numCols, cellsPerColumn = <NUM_LIT:4>,<EOL>initialPerm = initialPerm, connectedPerm = connectedPerm,<EOL>minThreshold = minThreshold,<EOL>newSynapseCount = newSynapseCount,<EOL>permanenceInc = permanenceInc,<EOL>permanenceDec = permanenceDec,<EOL>activationThreshold = activationThreshold,<EOL>globalDecay = globalDecay, maxAge=maxAge, burnIn = <NUM_LIT:1>,<EOL>seed=g_options.seed, verbosity=g_options.verbosity,<EOL>checkSynapseConsistency = checkSynapseConsistency,<EOL>pamLength = <NUM_LIT:1000>,<EOL>maxSegmentsPerCell = maxSegmentsPerCell,<EOL>maxSynapsesPerSegment = maxSynapsesPerSegment,<EOL>)<EOL>cppTM.retrieveLearningStates = True<EOL><DEDENT>else:<EOL><INDENT>cppTM = None<EOL><DEDENT>if g_options.verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>pyTM = BacktrackingTM(numberOfCols = numCols,<EOL>cellsPerColumn = <NUM_LIT:4>,<EOL>initialPerm = initialPerm,<EOL>connectedPerm = connectedPerm,<EOL>minThreshold = minThreshold,<EOL>newSynapseCount = newSynapseCount,<EOL>permanenceInc = permanenceInc,<EOL>permanenceDec = permanenceDec,<EOL>activationThreshold = activationThreshold,<EOL>globalDecay = globalDecay, maxAge=maxAge, burnIn = <NUM_LIT:1>,<EOL>seed=g_options.seed, verbosity=g_options.verbosity,<EOL>pamLength = <NUM_LIT:1000>,<EOL>maxSegmentsPerCell = maxSegmentsPerCell,<EOL>maxSynapsesPerSegment = maxSynapsesPerSegment,<EOL>)<EOL>return cppTM, pyTM<EOL>

Create an instance of the appropriate temporal memory. We isolate
        all parameters as constants specified here.


def _buildSL2TrainingSet(self, numOnes=<NUM_LIT:10>, numRepetitions= <NUM_LIT:10>):

numPatterns = <NUM_LIT:5><EOL>numCols = <NUM_LIT:2> * numPatterns * numOnes<EOL>halfCols = numPatterns * numOnes<EOL>numNoiseBits = numOnes<EOL>p = self._getSimplePatterns(numOnes, numPatterns)<EOL>numSequences = <NUM_LIT:3><EOL>indices = [<EOL>[<NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:4>],<EOL>[<NUM_LIT:4>, <NUM_LIT:3>, <NUM_LIT:2>, <NUM_LIT:1>, <NUM_LIT:0>],<EOL>[<NUM_LIT:2>, <NUM_LIT:0>, <NUM_LIT:4>, <NUM_LIT:1>, <NUM_LIT:3>],<EOL>]<EOL>trainingSequences = []<EOL>for i in range(numRepetitions*numSequences):<EOL><INDENT>sequence = []<EOL>for j in range(numPatterns):<EOL><INDENT>v = numpy.zeros(numCols, dtype='<STR_LIT>')<EOL>v[<NUM_LIT:0>:halfCols] = p[indices[i % numSequences][j]]<EOL>noiseIndices = (self._rgen.permutation(halfCols)<EOL>+ halfCols)[<NUM_LIT:0>:numNoiseBits]<EOL>v[noiseIndices] = <NUM_LIT:1><EOL>sequence.append(v)<EOL><DEDENT>trainingSequences.append(sequence)<EOL><DEDENT>testSequences = []<EOL>for i in range(numSequences):<EOL><INDENT>sequence = []<EOL>for j in range(numPatterns):<EOL><INDENT>v = numpy.zeros(numCols, dtype='<STR_LIT>')<EOL>v[<NUM_LIT:0>:halfCols] = p[indices[i % numSequences][j]]<EOL>sequence.append(v)<EOL><DEDENT>testSequences.append(sequence)<EOL><DEDENT>if g_options.verbosity > <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>self._printAllTrainingSequences(trainingSequences)<EOL>print("<STR_LIT>")<EOL>self._printAllTrainingSequences(testSequences)<EOL><DEDENT>return (trainingSequences, testSequences)<EOL>

Three simple sequences, composed of the same 5 static patterns. The left
        half of the vector contains the pattern elements, each with numOnes
        consecutive bits. The right half contains numOnes random bits.

        Sequence 1 is: p0, p1, p2, p3, p4
        Sequence 2 is: p4, p3, p2, p1, p0
        Sequence 3 is: p2, p0, p4, p1, p3

        The function returns a pair:

        trainingSequences:    A list containing numRepetitions instances of the
                              above sequences
        testSequence:         Clean test sequences with no noise on the right half


return Spec(engine_internal.Region.getSpecFromType(nodeType))<EOL>


engine_internal.Network.unregisterPyRegion(regionName)<EOL>

Unregisters a region from the internal list of regions

:param str regionName: The name of the region to unregister
    (ex: regionName=regionClass.__name__)


return self._region.__hash__()<EOL>


return Dimensions(tuple(self._region.getDimensions()))<EOL>


phases = engine_internal.UInt32Set(phases)<EOL>engine_internal.Network.setPhases(self, name, phases)<EOL>


def addRegion(self, name, nodeType, nodeParams):

engine_internal.Network.addRegion(self, name, nodeType, nodeParams)<EOL>return self._getRegions()[name]<EOL>


return self._region != other._region<EOL>


<EOL>engine_internal.Network.run(self, n)<EOL>


return getattr(self._region, method)()<EOL>

Auto forwarding of properties to get methods of internal region


regions = []<EOL>for region in list(self.regions.values()):<EOL><INDENT>if type(region.getSelf()) is regionClass:<EOL><INDENT>regions.append(region)<EOL><DEDENT><DEDENT>return regions<EOL>

Gets all region instances of a given class
(for example, nupic.regions.sp_region.SPRegion).


return self._region == other._region<EOL>


def makeRegion(name, r):<EOL><INDENT>"""<STR_LIT>"""<EOL>r = Region(r, self)<EOL>return r<EOL><DEDENT>regions = CollectionWrapper(engine_internal.Network.getRegions(self), makeRegion)<EOL>return regions<EOL>

Get the collection of regions in a network

        This is a tricky one. The collection of regions returned from
        from the internal network is a collection of internal regions.
        The desired collection is a collelcion of net.Region objects
        that also points to this network (net.network) and not to
        the internal network. To achieve that a CollectionWrapper
        class is used with a custom makeRegion() function (see bellow)
        as a value wrapper. The CollectionWrapper class wraps each value in the
        original collection with the result of the valueWrapper.


def getCallbacks(self, *args, **kwargs):

engine_internal.Network.getCallbacks(self, *args, **kwargs)<EOL>


(setter, getter) = self._getParameterMethods(paramName)<EOL>if getter is None:<EOL><INDENT>import exceptions<EOL>raise exceptions.Exception(<EOL>"<STR_LIT>"<EOL>% (paramName, self.name, self.type))<EOL><DEDENT>return getter(paramName)<EOL>


def getArrayType(self):<EOL><INDENT>"""<STR_LIT>"""<EOL>return self._dtype<EOL><DEDENT>if ref:<EOL><INDENT>assert size is None<EOL><DEDENT>index = basicTypes.index(dtype)<EOL>if index == -<NUM_LIT:1>:<EOL><INDENT>raise Exception('<STR_LIT>' + dtype)<EOL><DEDENT>if size and size <= <NUM_LIT:0>:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>suffix = '<STR_LIT>' if ref else '<STR_LIT>'<EOL>arrayFactory = getattr(engine_internal, dtype + suffix)<EOL>arrayFactory.getType = getArrayType<EOL>if size:<EOL><INDENT>a = arrayFactory(size)<EOL><DEDENT>else:<EOL><INDENT>a = arrayFactory()<EOL><DEDENT>a._dtype = basicTypes[index]<EOL>return a<EOL>

Factory function that creates typed Array or ArrayRef objects

    dtype - the data type of the array (as string).
      Supported types are: Byte, Int16, UInt16, Int32, UInt32, Int64, UInt64, Real32, Real64

    size - the size of the array. Must be positive integer.


seed = cls._hashCoordinate(coordinate)<EOL>rng = Random(seed)<EOL>return rng.getReal64()<EOL>

Returns the order for a coordinate.

@param coordinate (numpy.array) Coordinate
@return (float) A value in the interval [0, 1), representing the
                order of the coordinate


ranges = (xrange(n-radius, n+radius+<NUM_LIT:1>) for n in coordinate.tolist())<EOL>return numpy.array(list(itertools.product(*ranges)))<EOL>

Returns coordinates around given coordinate, within given radius.
Includes given coordinate.

@param coordinate (numpy.array) N-dimensional integer coordinate
@param radius (int) Radius around `coordinate`

@return (numpy.array) List of coordinates


def encodeIntoArray(self, inputData, output):

(coordinate, radius) = inputData<EOL>assert isinstance(radius, int), ("<STR_LIT>"<EOL>.format(radius, type(radius)))<EOL>neighbors = self._neighbors(coordinate, radius)<EOL>winners = self._topWCoordinates(neighbors, self.w)<EOL>bitFn = lambda coordinate: self._bitForCoordinate(coordinate, self.n)<EOL>indices = numpy.array([bitFn(w) for w in winners])<EOL>output[:] = <NUM_LIT:0><EOL>output[indices] = <NUM_LIT:1><EOL>

See `nupic.encoders.base.Encoder` for more information.

@param inputData (tuple) Contains coordinate (numpy.array, N-dimensional
                         integer coordinate) and radius (int)
@param output (numpy.array) Stores encoded SDR in this numpy array


@classmethod<EOL><INDENT>def _bitForCoordinate(cls, coordinate, n):<DEDENT>

seed = cls._hashCoordinate(coordinate)<EOL>rng = Random(seed)<EOL>return rng.getUInt32(n)<EOL>

Maps the coordinate to a bit in the SDR.

@param coordinate (numpy.array) Coordinate
@param n (int) The number of available bits in the SDR
@return (int) The index to a bit in the SDR


return numpy.array(self.getEncodedValues(input))<EOL>

See method description in :meth:`~.nupic.encoders.base.Encoder.getScalars`.

:param input: (datetime) representing the time being encoded

:returns: A numpy array of the corresponding scalar values in the following
          order: season, dayOfWeek, weekend, holiday, timeOfDay. Some of
          these fields might be omitted if they were not specified in the
          encoder.


names = []<EOL>def _formFieldName(encoder):<EOL><INDENT>if parentFieldName == '<STR_LIT>':<EOL><INDENT>return encoder.name<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' % (parentFieldName, encoder.name)<EOL><DEDENT><DEDENT>if self.seasonEncoder is not None:<EOL><INDENT>names.append(_formFieldName(self.seasonEncoder))<EOL><DEDENT>if self.dayOfWeekEncoder is not None:<EOL><INDENT>names.append(_formFieldName(self.dayOfWeekEncoder))<EOL><DEDENT>if self.customDaysEncoder is not None:<EOL><INDENT>names.append(_formFieldName(self.customDaysEncoder))<EOL><DEDENT>if self.weekendEncoder is not None:<EOL><INDENT>names.append(_formFieldName(self.weekendEncoder))<EOL><DEDENT>if self.holidayEncoder is not None:<EOL><INDENT>names.append(_formFieldName(self.holidayEncoder))<EOL><DEDENT>if self.timeOfDayEncoder is not None:<EOL><INDENT>names.append(_formFieldName(self.timeOfDayEncoder))<EOL><DEDENT>return names<EOL>


if input == SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return [None] * len(self.encoders)<EOL><DEDENT>else:<EOL><INDENT>assert isinstance(input, datetime.datetime)<EOL>scalars = self.getScalars(input)<EOL>result = []<EOL>for i in xrange(len(self.encoders)):<EOL><INDENT>(name, encoder, offset) = self.encoders[i]<EOL>result.extend(encoder.getBucketIndices(scalars[i]))<EOL><DEDENT>return result<EOL><DEDENT>


def encodeIntoArray(self, value, output):

denseInput = numpy.zeros(output.shape)<EOL>try:<EOL><INDENT>denseInput[value] = <NUM_LIT:1><EOL><DEDENT>except IndexError:<EOL><INDENT>if isinstance(value, numpy.ndarray):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>".format(<EOL>value.dtype))<EOL><DEDENT>raise<EOL><DEDENT>super(SparsePassThroughEncoder, self).encodeIntoArray(denseInput, output)<EOL>


if input == SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return numpy.array([None])<EOL><DEDENT>else:<EOL><INDENT>return numpy.array([self.categoryToIndex.get(input, <NUM_LIT:0>)])<EOL><DEDENT>


encoderResult = self.encoder.topDownCompute(encoded)[<NUM_LIT:0>]<EOL>value = encoderResult.value<EOL>categoryIndex = int(round(value))<EOL>category = self.indexToCategory[categoryIndex]<EOL>return EncoderResult(value=category, scalar=categoryIndex,<EOL>encoding=encoderResult.encoding)<EOL>


if self._bucketValues is None:<EOL><INDENT>numBuckets = len(self.encoder.getBucketValues())<EOL>self._bucketValues = []<EOL>for bucketIndex in range(numBuckets):<EOL><INDENT>self._bucketValues.append(self.getBucketInfo([bucketIndex])[<NUM_LIT:0>].value)<EOL><DEDENT><DEDENT>return self._bucketValues<EOL>


<EOL>if input == SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return [None]<EOL><DEDENT>else:<EOL><INDENT>return self.encoder.getBucketIndices(self.categoryToIndex.get(input, <NUM_LIT:0>))<EOL><DEDENT>


raise NotImplementedError()<EOL>

Encodes inputData and puts the encoded value into the numpy output array,
which is a 1-D array of length returned by :meth:`.getWidth`.

.. note:: The numpy output array is reused, so clear it before updating it.

:param inputData: Data to encode. This should be validated by the encoder.
:param output: numpy 1-D array of same length returned by
       :meth:`.getWidth`.


retVals = []<EOL>if self.encoders is not None:<EOL><INDENT>for (name, encoder, offset) in self.encoders:<EOL><INDENT>values = encoder.getBucketIndices(self._getInputValue(inputData, name))<EOL>retVals.extend(values)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert False, "<STR_LIT>""<STR_LIT>"<EOL><DEDENT>return retVals<EOL>

Returns an array containing the sub-field bucket indices for each sub-field
of the inputData. To get the associated field names for each of the buckets,
call :meth:`.getScalarNames`.

:param inputData: The data from the source. This is typically an object with
             members.
:return: array of bucket indices


retVals = numpy.array([])<EOL>if self.encoders is not None:<EOL><INDENT>for (name, encoder, offset) in self.encoders:<EOL><INDENT>values = encoder.getScalars(self._getInputValue(inputData, name))<EOL>retVals = numpy.hstack((retVals, values))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>retVals = numpy.hstack((retVals, inputData))<EOL><DEDENT>return retVals<EOL>

Returns a numpy array containing the sub-field scalar value(s) for
each sub-field of the ``inputData``. To get the associated field names for
each of the scalar values, call :meth:`.getScalarNames()`.

For a simple scalar encoder, the scalar value is simply the input unmodified.
For category encoders, it is the scalar representing the category string
that is passed in. For the datetime encoder, the scalar value is the
the number of seconds since epoch.

The intent of the scalar representation of a sub-field is to provide a
baseline for measuring error differences. You can compare the scalar value
of the inputData with the scalar value returned from :meth:`.topDownCompute`
on a top-down representation to evaluate prediction accuracy, for example.

:param inputData: The data from the source. This is typically an object with
             members
:return: array of scalar values


def closenessScores(self, expValues, actValues, fractional=True):

<EOL>if self.encoders is None:<EOL><INDENT>err = abs(expValues[<NUM_LIT:0>] - actValues[<NUM_LIT:0>])<EOL>if fractional:<EOL><INDENT>denom = max(expValues[<NUM_LIT:0>], actValues[<NUM_LIT:0>])<EOL>if denom == <NUM_LIT:0>:<EOL><INDENT>denom = <NUM_LIT:1.0><EOL><DEDENT>closeness = <NUM_LIT:1.0> - float(err)/denom<EOL>if closeness < <NUM_LIT:0>:<EOL><INDENT>closeness = <NUM_LIT:0><EOL><DEDENT><DEDENT>else:<EOL><INDENT>closeness = err<EOL><DEDENT>return numpy.array([closeness])<EOL><DEDENT>scalarIdx = <NUM_LIT:0><EOL>retVals = numpy.array([])<EOL>for (name, encoder, offset) in self.encoders:<EOL><INDENT>values = encoder.closenessScores(expValues[scalarIdx:], actValues[scalarIdx:],<EOL>fractional=fractional)<EOL>scalarIdx += len(values)<EOL>retVals = numpy.hstack((retVals, values))<EOL><DEDENT>return retVals<EOL>

Compute closeness scores between the expected scalar value(s) and actual
scalar value(s). The expected scalar values are typically those obtained
from the :meth:`.getScalars` method. The actual scalar values are typically
those returned from :meth:`.topDownCompute`.

This method returns one closeness score for each value in expValues (or
actValues which must be the same length). The closeness score ranges from
0 to 1.0, 1.0 being a perfect match and 0 being the worst possible match.

If this encoder is a simple, single field encoder, then it will expect
just 1 item in each of the ``expValues`` and ``actValues`` arrays.
Multi-encoders will expect 1 item per sub-encoder.

Each encoder type can define it's own metric for closeness. For example,
a category encoder may return either 1 or 0, if the scalar matches exactly
or not. A scalar encoder might return a percentage match, etc.

:param expValues: Array of expected scalar values, typically obtained from
                 :meth:`.getScalars`
:param actValues: Array of actual values, typically obtained from
                 :meth:`.topDownCompute`

:return: Array of closeness scores, one per item in expValues (or
         actValues).


mType = type(obj)<EOL>return mType is list or mType is tuple<EOL>

Helper function to determine if a function is a list or sequence.


raise Exception("<STR_LIT>")<EOL>

**Must be overridden by subclasses.**

Returns a list of items, one for each bucket defined by this encoder.
Each item is the value assigned to that bucket, this is the same as the
:attr:`.EncoderResult.value` that would be returned by
:meth:`.getBucketInfo` for that bucket and is in the same format as the
input that would be passed to :meth:`.encode`.

This call is faster than calling :meth:`.getBucketInfo` on each bucket
individually if all you need are the bucket values.

:return: list of items, each item representing the bucket value for that
         bucket.


def decode(self, encoded, parentFieldName='<STR_LIT>'):

fieldsDict = dict()<EOL>fieldsOrder = []<EOL>if parentFieldName == '<STR_LIT>':<EOL><INDENT>parentName = self.name<EOL><DEDENT>else:<EOL><INDENT>parentName = "<STR_LIT>" % (parentFieldName, self.name)<EOL><DEDENT>if self.encoders is not None:<EOL><INDENT>for i in range(len(self.encoders)):<EOL><INDENT>(name, encoder, offset) = self.encoders[i]<EOL>if i < len(self.encoders)-<NUM_LIT:1>:<EOL><INDENT>nextOffset = self.encoders[i+<NUM_LIT:1>][<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>nextOffset = self.width<EOL><DEDENT>fieldOutput = encoded[offset:nextOffset]<EOL>(subFieldsDict, subFieldsOrder) = encoder.decode(fieldOutput,<EOL>parentFieldName=parentName)<EOL>fieldsDict.update(subFieldsDict)<EOL>fieldsOrder.extend(subFieldsOrder)<EOL><DEDENT><DEDENT>return (fieldsDict, fieldsOrder)<EOL>

Takes an encoded output and does its best to work backwards and generate
the input that would have generated it.

In cases where the encoded output contains more ON bits than an input
would have generated, this routine will return one or more ranges of inputs
which, if their encoded outputs were ORed together, would produce the
target output. This behavior makes this method suitable for doing things
like generating a description of a learned coincidence in the SP, which
in many cases might be a union of one or more inputs.

If instead, you want to figure the *most likely* single input scalar value
that would have generated a specific encoded output, use the
:meth:`.topDownCompute` method.

If you want to pretty print the return value from this method, use the
:meth:`.decodedToStr` method.

:param encoded:      The encoded output that you want decode
:param parentFieldName: The name of the encoder which is our parent. This name
       is prefixed to each of the field names within this encoder to form the
       keys of the dict() in the retval.

:return: tuple(``fieldsDict``, ``fieldOrder``)

          ``fieldsDict`` is a dict() where the keys represent field names
          (only 1 if this is a simple encoder, > 1 if this is a multi
          or date encoder) and the values are the result of decoding each
          field. If there are  no bits in encoded that would have been
          generated by a field, it won't be present in the dict. The
          key of each entry in the dict is formed by joining the passed in
          parentFieldName with the child encoder name using a '.'.

          Each 'value' in ``fieldsDict`` consists of (ranges, desc), where
          ranges is a list of one or more (minVal, maxVal) ranges of
          input that would generate bits in the encoded output and 'desc'
          is a pretty print description of the ranges. For encoders like
          the category encoder, the 'desc' will contain the category
          names that correspond to the scalar values included in the
          ranges.

          ``fieldOrder`` is a list of the keys from ``fieldsDict``, in the
          same order as the fields appear in the encoded output.

          TODO: when we switch to Python 2.7 or 3.x, use OrderedDict

Example retvals for a scalar encoder:

.. code-block:: python

   {'amount':  ( [[1,3], [7,10]], '1-3, 7-10' )}
   {'amount':  ( [[2.5,2.5]],     '2.5'       )}

Example retval for a category encoder:

.. code-block:: python

   {'country': ( [[1,1], [5,6]], 'US, GB, ES' )}

Example retval for a multi encoder:

.. code-block:: python

   {'amount':  ( [[2.5,2.5]],     '2.5'       ),
    'country': ( [[1,1], [5,6]],  'US, GB, ES' )}


raise NotImplementedError()<EOL>

Should return the output width, in bits.

        :return: (int) output width in bits


def encodedBitDescription(self, bitOffset, formatted=False):

<EOL>(prevFieldName, prevFieldOffset) = (None, None)<EOL>description = self.getDescription()<EOL>for i in range(len(description)):<EOL><INDENT>(name, offset) = description[i]<EOL>if formatted:<EOL><INDENT>offset = offset + i<EOL>if bitOffset == offset-<NUM_LIT:1>:<EOL><INDENT>prevFieldName = "<STR_LIT>"<EOL>prevFieldOffset = bitOffset<EOL>break<EOL><DEDENT><DEDENT>if bitOffset < offset:<EOL><INDENT>break<EOL><DEDENT>(prevFieldName, prevFieldOffset) = (name, offset)<EOL><DEDENT>width = self.getDisplayWidth() if formatted else self.getWidth()<EOL>if prevFieldOffset is None or bitOffset > self.getWidth():<EOL><INDENT>raise IndexError("<STR_LIT>" % width)<EOL><DEDENT>return (prevFieldName, bitOffset - prevFieldOffset)<EOL>

Return a description of the given bit in the encoded output.
This will include the field name and the offset within the field.

:param bitOffset:      Offset of the bit to get the description of
:param formatted:      If True, the bitOffset is w.r.t. formatted output,
                      which includes separators
:return:             tuple(``fieldName``, ``offsetWithinField``)


if hasattr(self, '<STR_LIT>') andself._flattenedEncoderList is not None:<EOL><INDENT>return self._flattenedEncoderList<EOL><DEDENT>encoders = []<EOL>if self.encoders is not None:<EOL><INDENT>for (name, encoder, offset) in self.encoders:<EOL><INDENT>subEncoders = encoder.getEncoderList()<EOL>encoders.extend(subEncoders)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>encoders.append(self)<EOL><DEDENT>self._flattenedEncoderList = encoders<EOL>return encoders<EOL>

:return: a reference to each sub-encoder in this encoder. They are
         returned in the same order as they are for :meth:`.getScalarNames`
         and :meth:`.getScalars`.


def setFieldStats(self, fieldName, fieldStatistics):

pass<EOL>

This method is called by the model to set the statistics like min and
max for the underlying encoders if this information is available.

:param fieldName: name of the field this encoder is encoding, provided by
      :class:`~.nupic.encoders.multi.MultiEncoder`.

:param fieldStatistics: dictionary of dictionaries with the first level being
      the fieldname and the second index the statistic ie:
      ``fieldStatistics['pounds']['min']``


def _getInputValue(self, obj, fieldName):

if isinstance(obj, dict):<EOL><INDENT>if not fieldName in obj:<EOL><INDENT>knownFields = "<STR_LIT:U+002CU+0020>".join(<EOL>key for key in list(obj.keys()) if not key.startswith("<STR_LIT:_>")<EOL>)<EOL>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (<EOL>fieldName, knownFields, fieldName<EOL>)<EOL>)<EOL><DEDENT>return obj[fieldName]<EOL><DEDENT>else:<EOL><INDENT>return getattr(obj, fieldName)<EOL><DEDENT>

Gets the value of a given field from the input record


retVals = []<EOL>if self.encoders is not None:<EOL><INDENT>for name, encoders, offset in self.encoders:<EOL><INDENT>values = encoders.getEncodedValues(self._getInputValue(inputData, name))<EOL>if _isSequence(values):<EOL><INDENT>retVals.extend(values)<EOL><DEDENT>else:<EOL><INDENT>retVals.append(values)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if _isSequence(inputData):<EOL><INDENT>retVals.extend(inputData)<EOL><DEDENT>else:<EOL><INDENT>retVals.append(inputData)<EOL><DEDENT><DEDENT>return tuple(retVals)<EOL>

Returns the input in the same format as is returned by
:meth:`.topDownCompute`. For most encoder types, this is the same as the
input data. For instance, for scalar and category types, this corresponds to
the numeric and string values, respectively, from the inputs. For datetime
encoders, this returns the list of scalars for each of the sub-fields
(timeOfDay, dayOfWeek, etc.)

This method is essentially the same as :meth:`.getScalars` except that it
returns strings.

:param inputData: The input data in the format it is received from the data
                  source

:return: A list of values, in the same format and in the same order as they
         are returned by :meth:`.topDownCompute`.


def scalarsToStr(self, scalarValues, scalarNames=None):

if scalarNames is None:<EOL><INDENT>scalarNames = self.getScalarNames()<EOL><DEDENT>desc = '<STR_LIT>'<EOL>for (name, value) in zip(scalarNames, scalarValues):<EOL><INDENT>if len(desc) > <NUM_LIT:0>:<EOL><INDENT>desc += "<STR_LIT>" % (name, value)<EOL><DEDENT>else:<EOL><INDENT>desc += "<STR_LIT>" % (name, value)<EOL><DEDENT><DEDENT>return desc<EOL>

Return a pretty print string representing the return values from
:meth:`.getScalars` and :meth:`.getScalarNames`.

:param scalarValues: input values to encode to string
:param scalarNames: optional input of scalar names to convert. If None, gets
                   scalar names from :meth:`.getScalarNames`
:return: string representation of scalar values


<EOL>if hasattr(self, "<STR_LIT>"):<EOL><INDENT>self._learningEnabled = learningEnabled<EOL><DEDENT>

Set whether learning is enabled.

        :param learningEnabled: (bool) whether learning should be enabled


<EOL>if self.encoders is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>retVals = []<EOL>bucketOffset = <NUM_LIT:0><EOL>for i in range(len(self.encoders)):<EOL><INDENT>(name, encoder, offset) = self.encoders[i]<EOL>if encoder.encoders is not None:<EOL><INDENT>nextBucketOffset = bucketOffset + len(encoder.encoders)<EOL><DEDENT>else:<EOL><INDENT>nextBucketOffset = bucketOffset + <NUM_LIT:1><EOL><DEDENT>bucketIndices = buckets[bucketOffset:nextBucketOffset]<EOL>values = encoder.getBucketInfo(bucketIndices)<EOL>retVals.extend(values)<EOL>bucketOffset = nextBucketOffset<EOL><DEDENT>return retVals<EOL>

Returns a list of :class:`.EncoderResult` namedtuples describing the inputs
for each sub-field that correspond to the bucket indices passed in
``buckets``. To get the associated field names for each of the values, call
:meth:`.getScalarNames`.

:param buckets: The list of bucket indices, one for each sub-field encoder.
               These bucket indices for example may have been retrieved
               from the :meth:`.getBucketIndices` call.
:return: A list of :class:`.EncoderResult`.


if hasattr(self, '<STR_LIT>') andself._flattenedFieldTypeList is not None:<EOL><INDENT>return self._flattenedFieldTypeList<EOL><DEDENT>fieldTypes = []<EOL>for (name, encoder, offset) in self.encoders:<EOL><INDENT>subTypes = encoder.getDecoderOutputFieldTypes()<EOL>fieldTypes.extend(subTypes)<EOL><DEDENT>self._flattenedFieldTypeList = fieldTypes<EOL>return fieldTypes<EOL>

Returns a sequence of field types corresponding to the elements in the
decoded output field array.  The types are defined by
:class:`~nupic.data.field_meta.FieldMetaType`.

:return: list of :class:`~nupic.data.field_meta.FieldMetaType` objects


if inpt == SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>val = inpt<EOL>if val < self.minval:<EOL><INDENT>val = self.minval<EOL><DEDENT>elif val > self.maxval:<EOL><INDENT>val = self.maxval<EOL><DEDENT>scaledVal = math.log10(val)<EOL>return scaledVal<EOL><DEDENT>

Convert the input, which is in normal space, into log space


<EOL>scaledVal = self._getScaledValue(inpt)<EOL>if scaledVal is None:<EOL><INDENT>return [None]<EOL><DEDENT>else:<EOL><INDENT>return self.encoder.getBucketIndices(scaledVal)<EOL><DEDENT>


scaledResult = self.encoder.topDownCompute(encoded)[<NUM_LIT:0>]<EOL>scaledValue = scaledResult.value<EOL>value = math.pow(<NUM_LIT:10>, scaledValue)<EOL>return EncoderResult(value=value, scalar=value,<EOL>encoding = scaledResult.encoding)<EOL>


self.encoders.append((name, encoder, self.width))<EOL>for d in encoder.getDescription():<EOL><INDENT>self.description.append((d[<NUM_LIT:0>], d[<NUM_LIT:1>] + self.width))<EOL><DEDENT>self.width += encoder.getWidth()<EOL>

Adds one encoder.

:param name: (string) name of encoder, should be unique
:param encoder: (:class:`.Encoder`) the encoder to add


def encodeIntoArray(self, inputVal, outputVal):

if len(inputVal) != len(outputVal):<EOL><INDENT>raise ValueError("<STR_LIT>" % (<EOL>len(inputVal), len(outputVal)))<EOL><DEDENT>if self.w is not None and sum(inputVal) != self.w:<EOL><INDENT>raise ValueError("<STR_LIT>" % (<EOL>sum(inputVal), self.w))<EOL><DEDENT>outputVal[:] = inputVal[:]<EOL>if self.verbosity >= <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>", inputVal, "<STR_LIT>", outputVal)<EOL>print("<STR_LIT>", self.decodedToStr(self.decode(outputVal)))<EOL><DEDENT>


def closenessScores(self, expValues, actValues, **kwargs):

ratio = <NUM_LIT:1.0><EOL>esum = int(expValues.sum())<EOL>asum = int(actValues.sum())<EOL>if asum > esum:<EOL><INDENT>diff = asum - esum<EOL>if diff < esum:<EOL><INDENT>ratio = <NUM_LIT:1> - diff/float(esum)<EOL><DEDENT>else:<EOL><INDENT>ratio = <NUM_LIT:1>/float(diff)<EOL><DEDENT><DEDENT>olap = expValues & actValues<EOL>osum = int(olap.sum())<EOL>if esum == <NUM_LIT:0>:<EOL><INDENT>r = <NUM_LIT:0.0><EOL><DEDENT>else:<EOL><INDENT>r = osum/float(esum)<EOL><DEDENT>r = r * ratio<EOL>return numpy.array([r])<EOL>

Does a bitwise compare of the two bitmaps and returns a fractonal
value between 0 and 1 of how similar they are.

- ``1`` => identical
- ``0`` => no overlaping bits

``kwargs`` will have the keyword "fractional", which is assumed by this
encoder.


return numpy.array([<NUM_LIT:0>])<EOL>


return [<NUM_LIT:0>]<EOL>


overlap = <NUM_LIT><EOL>coordinatesPerTimestep = speed * self.timestep / self.scale<EOL>radius = int(round(float(coordinatesPerTimestep) / <NUM_LIT:2> * overlap))<EOL>minRadius = int(math.ceil((math.sqrt(self.w) - <NUM_LIT:1>) / <NUM_LIT:2>))<EOL>return max(radius, minRadius)<EOL>

Returns radius for given speed.

Tries to get the encodings of consecutive readings to be
adjacent with some overlap.

:param: speed (float) Speed (in meters per second)
:returns: (int) Radius for given speed


return numpy.array([<NUM_LIT:0>] * len(self.getDescription()))<EOL>

See `nupic.encoders.base.Encoder` for more information.