configPaths = []<EOL>if cls._configPaths is not None:<EOL><INDENT>return cls._configPaths<EOL><DEDENT>else:<EOL><INDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>configVar = os.environ['<STR_LIT>']<EOL>configPaths = configVar.split(os.pathsep)<EOL><DEDENT>return configPaths<EOL><DEDENT>

Return the list of paths to search for configuration files.

        :returns: (list) of paths


cls._configPaths = list(paths)<EOL>

Modify the paths we use to search for configuration files.

        :param paths: (list) of paths to search for config files.


<EOL>cls.readConfigFile(DEFAULT_CONFIG)<EOL>cls.readConfigFile(USER_CONFIG)<EOL>

Read in all standard configuration files


return _CustomConfigurationFileWrapper.getCustomDict()<EOL>

returns: (dict) containing all custom configuration properties.


@classmethod<EOL><INDENT>def setCustomProperty(cls, propertyName, value):<DEDENT>

cls.setCustomProperties({propertyName : value})<EOL>

Set a single custom setting and persist it to the custom configuration
store.

:param propertyName: (string) containing the name of the property to get
:param value: (object) value to set the property to


_getLogger().info("<STR_LIT>",<EOL>properties, traceback.format_stack())<EOL>_CustomConfigurationFileWrapper.edit(properties)<EOL>for propertyName, value in properties.items():<EOL><INDENT>cls.set(propertyName, value)<EOL><DEDENT>

Set multiple custom properties and persist them to the custom configuration
store.

:param properties: (dict) of property name/value pairs to set


<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=False)<EOL>

Clear all configuration properties from in-memory cache, but do NOT alter
the custom configuration file. Used in unit-testing.


_getLogger().info("<STR_LIT>"<EOL>"<STR_LIT>", traceback.format_stack())<EOL>super(Configuration, cls).clear()<EOL>_CustomConfigurationFileWrapper.clear(persistent=True)<EOL>

Clear all custom configuration settings and delete the persistent custom
configuration store.


cls.readConfigFile(_CustomConfigurationFileWrapper.customFileName)<EOL>

Loads custom configuration settings from their persistent storage.

.. warning :: DO NOT CALL THIS: It's typically not necessary to call this
   method directly. This method exists *solely* for the benefit of
   ``prepare_conf.py``, which needs to load configuration files selectively.


super(Configuration, cls)._readStdConfigFiles()<EOL>cls.loadCustomConfig()<EOL>

Intercept the _readStdConfigFiles call from our base config class to
        read in base and custom configuration settings.


if persistent:<EOL><INDENT>try:<EOL><INDENT>os.unlink(cls.getPath())<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != errno.ENOENT:<EOL><INDENT>_getLogger().exception("<STR_LIT>""<STR_LIT>", e.errno, cls.getPath())<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>cls._path = None<EOL>

If persistent is True, delete the temporary file

        Parameters:
        ----------------------------------------------------------------
        persistent: if True, custom configuration file is deleted


if not os.path.exists(cls.getPath()):<EOL><INDENT>return dict()<EOL><DEDENT>properties = Configuration._readConfigFile(os.path.basename(<EOL>cls.getPath()), os.path.dirname(cls.getPath()))<EOL>values = dict()<EOL>for propName in properties:<EOL><INDENT>if '<STR_LIT:value>' in properties[propName]:<EOL><INDENT>values[propName] = properties[propName]['<STR_LIT:value>']<EOL><DEDENT><DEDENT>return values<EOL>

Returns a dict of all temporary values in custom configuration file


copyOfProperties = copy(properties)<EOL>configFilePath = cls.getPath()<EOL>try:<EOL><INDENT>with open(configFilePath, '<STR_LIT:r>') as fp:<EOL><INDENT>contents = fp.read()<EOL><DEDENT><DEDENT>except IOError as e:<EOL><INDENT>if e.errno != errno.ENOENT:<EOL><INDENT>_getLogger().exception("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>e.errno, configFilePath, properties)<EOL>raise<EOL><DEDENT>contents = '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>elements = ElementTree.XML(contents)<EOL>ElementTree.tostring(elements)<EOL><DEDENT>except Exception as e:<EOL><INDENT>msg = "<STR_LIT>""<STR_LIT>" %(configFilePath, contents, type(e), e)<EOL>_getLogger().exception(msg)<EOL>raise RuntimeError(msg).with_traceback(sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>if elements.tag != '<STR_LIT>':<EOL><INDENT>e = "<STR_LIT>" %(elements.tag)<EOL>_getLogger().error(e)<EOL>raise RuntimeError(e)<EOL><DEDENT>for propertyItem in elements.findall('<STR_LIT>'):<EOL><INDENT>propInfo = dict((attr.tag, attr.text) for attr in propertyItem)<EOL>name = propInfo['<STR_LIT:name>']<EOL>if name in copyOfProperties:<EOL><INDENT>foundValues = propertyItem.findall('<STR_LIT>')<EOL>if len(foundValues) > <NUM_LIT:0>:<EOL><INDENT>foundValues[<NUM_LIT:0>].text = str(copyOfProperties.pop(name))<EOL>if not copyOfProperties:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>e = "<STR_LIT>" % (name,)<EOL>_getLogger().error(e)<EOL>raise RuntimeError(e)<EOL><DEDENT><DEDENT><DEDENT>for propertyName, value in copyOfProperties.items():<EOL><INDENT>newProp = ElementTree.Element('<STR_LIT>')<EOL>nameTag = ElementTree.Element('<STR_LIT:name>')<EOL>nameTag.text = propertyName<EOL>newProp.append(nameTag)<EOL>valueTag = ElementTree.Element('<STR_LIT:value>')<EOL>valueTag.text = str(value)<EOL>newProp.append(valueTag)<EOL>elements.append(newProp)<EOL><DEDENT>try:<EOL><INDENT>makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))<EOL>with open(configFilePath,'<STR_LIT:w>') as fp:<EOL><INDENT>fp.write(ElementTree.tostring(elements))<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>_getLogger().exception("<STR_LIT>"<EOL>"<STR_LIT>", properties, configFilePath)<EOL>raise<EOL><DEDENT>

Edits the XML configuration file with the parameters specified by
        properties

        Parameters:
        ----------------------------------------------------------------
        properties: dict of settings to be applied to the custom configuration store
                     (key is property name, value is value)


cls._path = os.path.join(os.environ['<STR_LIT>'],<EOL>cls.customFileName)<EOL>

Sets the path of the custom configuration file


if cls._path is None:<EOL><INDENT>cls._setPath()<EOL><DEDENT>return cls._path<EOL>

Get the path of the custom configuration file


logger = (logger if logger is not None else logging.getLogger(__name__))<EOL>def exceptionLoggingDecorator(func):<EOL><INDENT>@functools.wraps(func)<EOL>def exceptionLoggingWrap(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>except:<EOL><INDENT>logger.exception(<EOL>"<STR_LIT>",<EOL>sys.exc_info()[<NUM_LIT:1>], func, '<STR_LIT>'.join(traceback.format_stack()), )<EOL>raise<EOL><DEDENT><DEDENT>return exceptionLoggingWrap<EOL><DEDENT>return exceptionLoggingDecorator<EOL>

Returns a closure suitable for use as function/method decorator for
    logging exceptions that leave the scope of the decorated function. Exceptions
    are logged at ERROR level.

    logger:    user-supplied logger instance. Defaults to logging.getLogger.

    Usage Example:
      NOTE: logging must be initialized *before* any loggers are created, else
        there will be no output; see nupic.support.initLogging()

      @logExceptions()
      def myFunctionFoo():
          ...
          raise RuntimeError("something bad happened")
          ...


def entryExitLoggingDecorator(func):<EOL><INDENT>@functools.wraps(func)<EOL>def entryExitLoggingWrap(*args, **kwargs):<EOL><INDENT>if entryExitLogLevel is None:<EOL><INDENT>enabled = False<EOL><DEDENT>else:<EOL><INDENT>logger = getLoggerCallback()<EOL>enabled = logger.isEnabledFor(entryExitLogLevel)<EOL><DEDENT>if not enabled:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>funcName = str(func)<EOL>if logArgs:<EOL><INDENT>argsRepr = '<STR_LIT:U+002CU+0020>'.join(<EOL>[repr(a) for a in args] +<EOL>['<STR_LIT>' % (k,v,) for k,v in kwargs.items()])<EOL><DEDENT>else:<EOL><INDENT>argsRepr = '<STR_LIT>'<EOL><DEDENT>logger.log(<EOL>entryExitLogLevel, "<STR_LIT>", funcName, argsRepr,<EOL>'<STR_LIT>' if not logTraceback else '<STR_LIT>' + repr(traceback.format_stack()))<EOL>try:<EOL><INDENT>return func(*args, **kwargs)<EOL><DEDENT>finally:<EOL><INDENT>logger.log(<EOL>entryExitLogLevel, "<STR_LIT>", funcName, argsRepr,<EOL>'<STR_LIT>' if not logTraceback else '<STR_LIT>' + repr(traceback.format_stack()))<EOL><DEDENT><DEDENT>return entryExitLoggingWrap<EOL><DEDENT>return entryExitLoggingDecorator<EOL>

Returns a closure suitable for use as function/method decorator for
    logging entry/exit of function/method.

    getLoggerCallback:    user-supplied callback function that takes no args and
                            returns the logger instance to use for logging.
    entryExitLogLevel:    Log level for logging entry/exit of decorated function;
                            e.g., logging.DEBUG; pass None to disable entry/exit
                            logging.
    logArgs:              If True, also log args
    logTraceback:         If True, also log Traceback information

    Usage Examples:
      NOTE: logging must be initialized *before* any loggers are created, else
        there will be no output; see nupic.support.initLogging()

      @logEntryExit()
      def myFunctionBar():
          ...

      @logEntryExit(logTraceback=True)
      @logExceptions()
      def myFunctionGamma():
          ...
          raise RuntimeError("something bad happened")
          ...


def retry(timeoutSec, initialRetryDelaySec, maxRetryDelaySec,<EOL>retryExceptions=(Exception,),<EOL>retryFilter=lambda e, args, kwargs: True,<EOL>logger=None, clientLabel="<STR_LIT>"):

assert initialRetryDelaySec > <NUM_LIT:0>, str(initialRetryDelaySec)<EOL>assert timeoutSec >= <NUM_LIT:0>, str(timeoutSec)<EOL>assert maxRetryDelaySec >= initialRetryDelaySec,"<STR_LIT>" % (maxRetryDelaySec, initialRetryDelaySec)<EOL>assert isinstance(retryExceptions, tuple), (<EOL>"<STR_LIT>") % (type(retryExceptions),)<EOL>if logger is None:<EOL><INDENT>logger = logging.getLogger(__name__)<EOL><DEDENT>def retryDecorator(func):<EOL><INDENT>@functools.wraps(func)<EOL>def retryWrap(*args, **kwargs):<EOL><INDENT>numAttempts = <NUM_LIT:0><EOL>delaySec = initialRetryDelaySec<EOL>startTime = time.time()<EOL>while True:<EOL><INDENT>numAttempts += <NUM_LIT:1><EOL>try:<EOL><INDENT>result = func(*args, **kwargs)<EOL><DEDENT>except retryExceptions as e:<EOL><INDENT>if not retryFilter(e, args, kwargs):<EOL><INDENT>if logger.isEnabledFor(logging.DEBUG):<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', clientLabel, func,<EOL>'<STR_LIT>'.join(traceback.format_stack()), exc_info=True)<EOL><DEDENT>raise<EOL><DEDENT>now = time.time()<EOL>if now < startTime:<EOL><INDENT>startTime = now<EOL><DEDENT>if (now - startTime) >= timeoutSec:<EOL><INDENT>logger.exception(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', clientLabel, timeoutSec, numAttempts, func,<EOL>'<STR_LIT>'.join(traceback.format_stack()))<EOL>raise<EOL><DEDENT>if numAttempts == <NUM_LIT:1>:<EOL><INDENT>logger.warning(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', clientLabel, func, delaySec,<EOL>timeoutSec, '<STR_LIT>'.join(traceback.format_stack()), exc_info=True)<EOL><DEDENT>else:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>clientLabel, func, numAttempts, delaySec, timeoutSec,<EOL>'<STR_LIT>'.join(traceback.format_stack()), exc_info=True)<EOL><DEDENT>time.sleep(delaySec)<EOL>delaySec = min(delaySec*<NUM_LIT:2>, maxRetryDelaySec)<EOL><DEDENT>else:<EOL><INDENT>if numAttempts > <NUM_LIT:1>:<EOL><INDENT>logger.info('<STR_LIT>',<EOL>clientLabel, func, numAttempts)<EOL><DEDENT>return result<EOL><DEDENT><DEDENT><DEDENT>return retryWrap<EOL><DEDENT>return retryDecorator<EOL>

Returns a closure suitable for use as function/method decorator for
    retrying a function being decorated.

    timeoutSec:           How many seconds from time of initial call to stop
                          retrying (floating point); 0 = no retries
    initialRetryDelaySec: Number of seconds to wait for first retry.
                          Subsequent retries will occur at geometrically
                          doubling intervals up to a maximum interval of
                          maxRetryDelaySec (floating point)
    maxRetryDelaySec:     Maximum amount of seconds to wait between retries
                          (floating point)
    retryExceptions:      A tuple (must be a tuple) of exception classes that,
                          including their subclasses, should trigger retries;
                          Default: any Exception-based exception will trigger
                          retries
    retryFilter:          Optional filter function used to further filter the
                          exceptions in the retryExceptions tuple; called if the
                          current exception meets the retryExceptions criteria:
                          takes the current exception instance, args, and kwargs
                          that were passed to the decorated function, and returns
                          True to retry, False to allow the exception to be
                          re-raised without retrying. Default: permits any
                          exception that matches retryExceptions to be retried.
    logger:               User-supplied logger instance to use for logging.
                          None=defaults to logging.getLogger(__name__).

    Usage Example:
      NOTE: logging must be initialized *before* any loggers are created, else
        there will be no output; see nupic.support.initLogging()

      _retry = retry(timeoutSec=300, initialRetryDelaySec=0.2,
                     maxRetryDelaySec=10, retryExceptions=[socket.error])
      @_retry
      def myFunctionFoo():
          ...
          raise RuntimeError("something bad happened")
          ...


assert os.path.isabs(absDirPath)<EOL>try:<EOL><INDENT>os.makedirs(absDirPath)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != os.errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return absDirPath<EOL>

Makes directory for the given directory path with default permissions.
    If the directory already exists, it is treated as success.

    :param absDirPath: (string) absolute path of the directory to create.
    :raises: OSError if directory creation fails
    :returns: (string) absolute path provided


@staticmethod<EOL><INDENT>def _openStream(dataUrl,<EOL>isBlocking,  <EOL>maxTimeout,  <EOL>bookmark,<EOL>firstRecordIdx):<DEDENT>

filePath = dataUrl[len(FILE_PREF):]<EOL>if not os.path.isabs(filePath):<EOL><INDENT>filePath = os.path.join(os.getcwd(), filePath)<EOL><DEDENT>return FileRecordStream(streamID=filePath,<EOL>write=False,<EOL>bookmark=bookmark,<EOL>firstRecord=firstRecordIdx)<EOL>

Open the underlying file stream
        This only supports 'file://' prefixed paths.

        :returns: record stream instance
        :rtype: FileRecordStream


return self._recordStore.close()<EOL>


<EOL>while True:<EOL><INDENT>if self._sourceLastRecordIdx is not None  andself._recordStore.getNextRecordIdx() >= self._sourceLastRecordIdx:<EOL><INDENT>preAggValues = None                             <EOL>bookmark = self._recordStore.getBookmark()<EOL><DEDENT>else:<EOL><INDENT>preAggValues = self._recordStore.getNextRecord()<EOL>bookmark = self._recordStore.getBookmark()<EOL><DEDENT>if preAggValues == ():  <EOL><INDENT>if self._eofOnTimeout:<EOL><INDENT>preAggValues = None  <EOL><DEDENT>else:<EOL><INDENT>return preAggValues  <EOL><DEDENT><DEDENT>self._logger.debug('<STR_LIT>',<EOL>self._recordStore.getNextRecordIdx()-<NUM_LIT:1>, preAggValues)<EOL>(fieldValues, aggBookmark) = self._aggregator.next(preAggValues, bookmark)<EOL>if fieldValues is not None:<EOL><INDENT>self._aggBookmark = aggBookmark<EOL><DEDENT>if preAggValues is None and fieldValues is None:<EOL><INDENT>return None<EOL><DEDENT>if fieldValues is not None:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if self._needFieldsFiltering:<EOL><INDENT>values = []<EOL>srcDict = dict(zip(self._recordStoreFieldNames, fieldValues))<EOL>for name in self._streamFieldNames:<EOL><INDENT>values.append(srcDict[name])<EOL><DEDENT>fieldValues = values<EOL><DEDENT>if self._writer is not None:<EOL><INDENT>self._writer.appendRecord(fieldValues)<EOL><DEDENT>self._recordCount += <NUM_LIT:1><EOL>self._logger.debug('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self._recordCount-<NUM_LIT:1>, fieldValues, self._aggBookmark)<EOL>return fieldValues<EOL>

Returns combined data from all sources (values only).

        :returns: None on EOF; empty sequence on timeout.


inputRowCountAfterAggregation = <NUM_LIT:0><EOL>while True:<EOL><INDENT>record = self.getNextRecord()<EOL>if record is None:<EOL><INDENT>return inputRowCountAfterAggregation<EOL><DEDENT>inputRowCountAfterAggregation += <NUM_LIT:1><EOL>if inputRowCountAfterAggregation > <NUM_LIT>:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT><DEDENT>

Iterates through stream to calculate total records after aggregation.
This will alter the bookmark state.


return self._recordCount<EOL>

:returns: the index of the record that will be read next from
          :meth:`getNextRecord`.


return self._recordStore.recordsExistAfter(bookmark)<EOL>

:returns: True if there are records left after the  bookmark.


return self._aggMonthsAndSeconds<EOL>

Returns the aggregation period of the record stream as a dict
        containing 'months' and 'seconds'. The months is always an integer and
        seconds is a floating point. Only one is allowed to be non-zero at a
        time.

        Will return the aggregation period from this call. This call is
        used by the :meth:`nupic.data.record_stream.RecordStream.getNextRecordDict`
        method to assign a record number to a record given its timestamp and the
        aggregation interval.

        :returns: aggregationPeriod (as a dict) where:

                  - ``months``: number of months in aggregation period
                  - ``seconds``: number of seconds in aggregation period
                    (as a float)


return [f.name for f in self._streamFields]<EOL>

Returns all fields in all inputs (list of plain names).

.. note:: currently, only one input is supported


return self._streamFields<EOL>

:returns: a sequence of :class:`nupic.data.fieldmeta.FieldMetaInfo` for each
          field in the stream.


return self._aggBookmark<EOL>

:returns: a bookmark to the current position


<EOL>recordStoreStats = self._recordStore.getStats()<EOL>streamStats = dict()<EOL>for (key, values) in recordStoreStats.items():<EOL><INDENT>fieldStats = dict(zip(self._recordStoreFieldNames, values))<EOL>streamValues = []<EOL>for name in self._streamFieldNames:<EOL><INDENT>streamValues.append(fieldStats[name])<EOL><DEDENT>streamStats[key] = streamValues<EOL><DEDENT>return streamStats<EOL>

TODO: This method needs to be enhanced to get the stats on the *aggregated*
records.

:returns: stats (like min and max values of the fields).


return self._recordStore.getError()<EOL>


Saves specified error in the stream.

        :param error: to save


return self._recordStore.isCompleted()<EOL>

:returns: True if all records have been read.


<EOL>self._recordStore.setCompleted(completed)<EOL>

Marks the stream completed (True or False)

:param completed: (bool) is completed or not


self._recordStore.setTimeout(timeout)<EOL>

Set the read timeout.

        :param timeout: (float or int) timeout length


<EOL>dictPairs = [(original, updates)]<EOL>while len(dictPairs) > <NUM_LIT:0>:<EOL><INDENT>original, updates = dictPairs.pop()<EOL>for k, v in updates.items():<EOL><INDENT>if k in original and isinstance(original[k], dict) and isinstance(v, dict):<EOL><INDENT>dictPairs.append((original[k], v))<EOL><DEDENT>else:<EOL><INDENT>original[k] = v<EOL><DEDENT><DEDENT><DEDENT>

Recursively updates the values in original with the values from updates.


remainingDicts = [(d, ())]<EOL>while len(remainingDicts) > <NUM_LIT:0>:<EOL><INDENT>current, prevKeys = remainingDicts.pop()<EOL>for k, v in current.items():<EOL><INDENT>keys = prevKeys + (k,)<EOL>if isinstance(v, dict):<EOL><INDENT>remainingDicts.insert(<NUM_LIT:0>, (v, keys))<EOL><DEDENT>else:<EOL><INDENT>f(v, keys)<EOL><DEDENT><DEDENT><DEDENT>

Recursively applies f to the values in dict d.

    Args:
      d: The dict to recurse over.
      f: A function to apply to values in d that takes the value and a list of
          keys from the root of the dict to the value.


differences = dictDiff(da, db)<EOL>if not differences:<EOL><INDENT>return differences<EOL><DEDENT>if differences['<STR_LIT>']:<EOL><INDENT>print("<STR_LIT>" % differences['<STR_LIT>'])<EOL><DEDENT>if differences['<STR_LIT>']:<EOL><INDENT>print("<STR_LIT>" % differences['<STR_LIT>'])<EOL><DEDENT>for key in differences['<STR_LIT>']:<EOL><INDENT>print("<STR_LIT>" % (key, key))<EOL>print("<STR_LIT>" % (key, da[key]))<EOL>print("<STR_LIT>" % (key, db[key]))<EOL><DEDENT>return differences<EOL>

Compares two python dictionaries at the top level and report differences,
    if any, to stdout

    da:             first dictionary
    db:             second dictionary

    Returns:        The same value as returned by dictDiff() for the given args


different = False<EOL>resultDict = dict()<EOL>resultDict['<STR_LIT>'] = set(da) - set(db)<EOL>if resultDict['<STR_LIT>']:<EOL><INDENT>different = True<EOL><DEDENT>resultDict['<STR_LIT>'] = set(db) - set(da)<EOL>if resultDict['<STR_LIT>']:<EOL><INDENT>different = True<EOL><DEDENT>resultDict['<STR_LIT>'] = []<EOL>for key in (set(da) - resultDict['<STR_LIT>']):<EOL><INDENT>comparisonResult = da[key] == db[key]<EOL>if isinstance(comparisonResult, bool):<EOL><INDENT>isEqual = comparisonResult<EOL><DEDENT>else:<EOL><INDENT>isEqual = comparisonResult.all()<EOL><DEDENT>if not isEqual:<EOL><INDENT>resultDict['<STR_LIT>'].append(key)<EOL>different = True<EOL><DEDENT><DEDENT>assert (((resultDict['<STR_LIT>'] or resultDict['<STR_LIT>'] or<EOL>resultDict['<STR_LIT>']) and different) or not different)<EOL>return resultDict if different else None<EOL>

Compares two python dictionaries at the top level and return differences

    da:             first dictionary
    db:             second dictionary

    Returns:        None if dictionaries test equal; otherwise returns a
                    dictionary as follows:
                    {
                      'inAButNotInB':
                          <sequence of keys that are in da but not in db>
                      'inBButNotInA':
                          <sequence of keys that are in db but not in da>
                      'differentValues':
                          <sequence of keys whose corresponding values differ
                           between da and db>
                    }


def sort(filename, key, outputFile, fields=None, watermark=<NUM_LIT> * <NUM_LIT> * <NUM_LIT:100>):

if fields is not None:<EOL><INDENT>assert set(key).issubset(set([f[<NUM_LIT:0>] for f in fields]))<EOL><DEDENT>with FileRecordStream(filename) as f:<EOL><INDENT>if fields:<EOL><INDENT>fieldNames = [ff[<NUM_LIT:0>] for ff in fields]<EOL>indices = [f.getFieldNames().index(name) for name in fieldNames]<EOL>assert len(indices) == len(fields)<EOL><DEDENT>else:<EOL><INDENT>fileds = f.getFields()<EOL>fieldNames = f.getFieldNames()<EOL>indices = None<EOL><DEDENT>key = [fieldNames.index(name) for name in key]<EOL>chunk = <NUM_LIT:0><EOL>records = []<EOL>for i, r in enumerate(f):<EOL><INDENT>if indices:<EOL><INDENT>temp = []<EOL>for i in indices:<EOL><INDENT>temp.append(r[i])<EOL><DEDENT>r = temp<EOL><DEDENT>records.append(r)<EOL>available_memory = psutil.avail_phymem()<EOL>if available_memory < watermark:<EOL><INDENT>_sortChunk(records, key, chunk, fields)<EOL>records = []<EOL>chunk += <NUM_LIT:1><EOL><DEDENT><DEDENT>if len(records) > <NUM_LIT:0>:<EOL><INDENT>_sortChunk(records, key, chunk, fields)<EOL>chunk += <NUM_LIT:1><EOL><DEDENT>_mergeFiles(key, chunk, outputFile, fields)<EOL><DEDENT>

Sort a potentially big file

    filename - the input file (standard File format)
    key - a list of field names to sort by
    outputFile - the name of the output file
    fields - a list of fields that should be included (all fields if None)
    watermark - when available memory goes bellow the watermark create a new chunk

    sort() works by reading as records from the file into memory
    and calling _sortChunk() on each chunk. In the process it gets
    rid of unneeded fields if any. Once all the chunks have been sorted and
    written to chunk files it calls _merge() to merge all the chunks into a
    single sorted file.

    Note, that sort() gets a key that contains field names, which it converts
    into field indices for _sortChunk() becuase _sortChunk() doesn't need to know
    the field name.

    sort() figures out by itself how many chunk files to use by reading records
    from the file until the low watermark value of availabel memory is hit and
    then it sorts the current records, generates a chunk file, clears the sorted
    records and starts on a new chunk.

    The key field names are turned into indices


def _sortChunk(records, key, chunkIndex, fields):

title(additional='<STR_LIT>' % (str(key), chunkIndex))<EOL>assert len(records) > <NUM_LIT:0><EOL>records.sort(key=itemgetter(*key))<EOL>if chunkIndex is not None:<EOL><INDENT>filename = '<STR_LIT>' % chunkIndex<EOL>with FileRecordStream(filename, write=True, fields=fields) as o:<EOL><INDENT>for r in records:<EOL><INDENT>o.appendRecord(r)<EOL><DEDENT><DEDENT>assert os.path.getsize(filename) > <NUM_LIT:0><EOL><DEDENT>return records<EOL>

Sort in memory chunk of records

    records - a list of records read from the original dataset
    key - a list of indices to sort the records by
    chunkIndex - the index of the current chunk

    The records contain only the fields requested by the user.

    _sortChunk() will write the sorted records to a standard File
    named "chunk_<chunk index>.csv" (chunk_0.csv, chunk_1.csv,...).


def _mergeFiles(key, chunkCount, outputFile, fields):

title()<EOL>files = [FileRecordStream('<STR_LIT>' % i) for i in range(chunkCount)]<EOL>with FileRecordStream(outputFile, write=True, fields=fields) as o:<EOL><INDENT>files = [FileRecordStream('<STR_LIT>' % i) for i in range(chunkCount)]<EOL>records = [f.getNextRecord() for f in files]<EOL>while not all(r is None for r in records):<EOL><INDENT>indices = [i for i,r in enumerate(records) if r is not None]<EOL>records = [records[i] for i in indices]<EOL>files = [files[i] for i in indices]<EOL>r = min(records, key=itemgetter(*key))<EOL>o.appendRecord(r)<EOL>index = records.index(r)<EOL>records[index] = files[index].getNextRecord()<EOL><DEDENT><DEDENT>for i, f in enumerate(files):<EOL><INDENT>f.close()<EOL>os.remove('<STR_LIT>' % i)<EOL><DEDENT>

Merge sorted chunk files into a sorted output file

    chunkCount - the number of available chunk files
    outputFile the name of the sorted output file

    _mergeFiles()


if self._file is not None:<EOL><INDENT>self._file.close()<EOL>self._file = None<EOL><DEDENT>


<EOL>super(FileRecordStream, self).rewind()<EOL>self.close()<EOL>self._file = open(self._filename, self._mode)<EOL>self._reader = csv.reader(self._file, dialect="<STR_LIT>")<EOL>self._reader.next()<EOL>self._reader.next()<EOL>self._reader.next()<EOL>self._recordCount = <NUM_LIT:0><EOL>

Put us back at the beginning of the file again.


assert self._file is not None<EOL>assert self._mode == self._FILE_READ_MODE<EOL>try:<EOL><INDENT>line = self._reader.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>if self.rewindAtEOF:<EOL><INDENT>if self._recordCount == <NUM_LIT:0>:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>" % self._filename)<EOL><DEDENT>self.rewind()<EOL>line = self._reader.next()<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>self._recordCount += <NUM_LIT:1><EOL>record = []<EOL>for i, f in enumerate(line):<EOL><INDENT>if f in self._missingValues:<EOL><INDENT>record.append(SENTINEL_VALUE_FOR_MISSING_DATA)<EOL><DEDENT>else:<EOL><INDENT>record.append(self._adapters[i](f))<EOL><DEDENT><DEDENT>return record<EOL>

Returns next available data record from the file.

        :returns: a data row (a list or tuple) if available; None, if no more
                  records in the table (End of Stream - EOS); empty sequence (list
                  or tuple) when timing out while waiting for the next record.


assert self._file is not None<EOL>assert self._mode == self._FILE_WRITE_MODE<EOL>assert isinstance(record, (list, tuple)),"<STR_LIT>" + repr(type(record))<EOL>assert len(record) == self._fieldCount,"<STR_LIT>" % (len(record), self._fieldCount)<EOL>if self._recordCount == <NUM_LIT:0>:<EOL><INDENT>names, types, specials = zip(*self.getFields())<EOL>for line in names, types, specials:<EOL><INDENT>self._writer.writerow(line)<EOL><DEDENT><DEDENT>self._updateSequenceInfo(record)<EOL>line = [self._adapters[i](f) for i, f in enumerate(record)]<EOL>self._writer.writerow(line)<EOL>self._recordCount += <NUM_LIT:1><EOL>

Saves the record in the underlying csv file.

:param record: a list of Python objects that will be string-ified


def appendRecords(self, records, progressCB=None):

for record in records:<EOL><INDENT>self.appendRecord(record)<EOL>if progressCB is not None:<EOL><INDENT>progressCB()<EOL><DEDENT><DEDENT>

Saves multiple records in the underlying storage.

:param records: array of records as in
                :meth:`~.FileRecordStream.appendRecord`
:param progressCB: (function) callback to report progress


if self._write and self._recordCount==<NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>rowDict = dict(filepath=os.path.realpath(self._filename),<EOL>currentRow=self._recordCount)<EOL>return json.dumps(rowDict)<EOL>

Gets a bookmark or anchor to the current position.

:returns: an anchor to the current position in the data. Passing this
          anchor to a constructor makes the current position to be the first
          returned record.


return (self.getDataRowCount() - self.getNextRecordIdx()) > <NUM_LIT:0><EOL>

Returns whether there are more records from current position. ``bookmark``
is not used in this implementation.

:return: True if there are records left after current position.


self._file.seek(self._getTotalLineCount() - numRecords)<EOL>return self.getBookmark()<EOL>

Seeks to ``numRecords`` from the end and returns a bookmark to the new
position.

:param numRecords: how far to seek from end of file.
:return: bookmark to desired location.


self.rewindAtEOF = autoRewind<EOL>

Controls whether :meth:`~.FileRecordStream.getNextRecord` should
automatically rewind the source when EOF is reached.

:param autoRewind: (bool)

    - if True, :meth:`~.FileRecordStream.getNextRecord` will automatically rewind
      the source on EOF.
    - if False, :meth:`~.FileRecordStream.getNextRecord` will not automatically
      rewind the source on EOF.


<EOL>if self._stats == None:<EOL><INDENT>assert self._mode == self._FILE_READ_MODE<EOL>inFile = open(self._filename, self._FILE_READ_MODE)<EOL>reader = csv.reader(inFile, dialect="<STR_LIT>")<EOL>names = [n.strip() for n in reader.next()]<EOL>types = [t.strip() for t in reader.next()]<EOL>reader.next()<EOL>self._stats = dict()<EOL>self._stats['<STR_LIT>'] = []<EOL>self._stats['<STR_LIT>'] = []<EOL>for i in xrange(len(names)):<EOL><INDENT>self._stats['<STR_LIT>'].append(None)<EOL>self._stats['<STR_LIT>'].append(None)<EOL><DEDENT>while True:<EOL><INDENT>try:<EOL><INDENT>line = reader.next()<EOL>for i, f in enumerate(line):<EOL><INDENT>if (len(types) > i and<EOL>types[i] in [FieldMetaType.integer, FieldMetaType.float] and<EOL>f not in self._missingValues):<EOL><INDENT>value = self._adapters[i](f)<EOL>if self._stats['<STR_LIT>'][i] == None orself._stats['<STR_LIT>'][i] < value:<EOL><INDENT>self._stats['<STR_LIT>'][i] = value<EOL><DEDENT>if self._stats['<STR_LIT>'][i] == None orself._stats['<STR_LIT>'][i] > value:<EOL><INDENT>self._stats['<STR_LIT>'][i] = value<EOL><DEDENT><DEDENT><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>return self._stats<EOL>

Parse the file using dedicated reader and collect fields stats. Never
called if user of :class:`~.FileRecordStream` does not invoke
:meth:`~.FileRecordStream.getStats` method.

:returns:
      a dictionary of stats. In the current implementation, min and max
      fields are supported. Example of the return dictionary is:

      .. code-block:: python

         {
           'min' : [f1_min, f2_min, None, None, fn_min],
           'max' : [f1_max, f2_max, None, None, fn_max]
         }

      (where fx_min/fx_max are set for scalar fields, or None if not)


self._stats = None<EOL>


return None<EOL>

Not implemented. CSV file version does not provide storage for the error
information


return<EOL>


return True<EOL>

Not implemented. CSV file is always considered completed.


return<EOL>

Not implemented: CSV file is always considered completed, nothing to do.


return [f.name for f in self._fields]<EOL>

:returns: (list) field names associated with the data.


if self._fields is None:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return copy.copy(self._fields)<EOL><DEDENT>

:returns: a sequence of :class:`~.FieldMetaInfo`
          ``name``/``type``/``special`` tuples for each field in the stream.


<EOL>newSequence = False<EOL>sequenceId = (r[self._sequenceIdIdx]<EOL>if self._sequenceIdIdx is not None else None)<EOL>if sequenceId != self._currSequence:<EOL><INDENT>if sequenceId in self._sequences:<EOL><INDENT>raise Exception('<STR_LIT>' %(sequenceId, r))<EOL><DEDENT>self._sequences.add(self._currSequence)<EOL>self._currSequence = sequenceId<EOL>if self._resetIdx:<EOL><INDENT>assert r[self._resetIdx] == <NUM_LIT:1><EOL><DEDENT>newSequence = True<EOL><DEDENT>else:<EOL><INDENT>reset = False<EOL>if self._resetIdx:<EOL><INDENT>reset = r[self._resetIdx]<EOL>if reset == <NUM_LIT:1>:<EOL><INDENT>newSequence = True<EOL><DEDENT><DEDENT><DEDENT>if not newSequence:<EOL><INDENT>if self._timeStampIdx and self._currTime is not None:<EOL><INDENT>t = r[self._timeStampIdx]<EOL>if t < self._currTime:<EOL><INDENT>raise Exception('<STR_LIT>' % r)<EOL><DEDENT><DEDENT><DEDENT>if self._timeStampIdx:<EOL><INDENT>self._currTime = r[self._timeStampIdx]<EOL><DEDENT>

Keep track of sequence and make sure time goes forward

        Check if the current record is the beginning of a new sequence
        A new sequence starts in 2 cases:

        1. The sequence id changed (if there is a sequence id field)
        2. The reset field is 1 (if there is a reset field)

        Note that if there is no sequenceId field or resetId field then the entire
        dataset is technically one big sequence. The function will not return True
        for the first record in this case. This is Ok because it is important to
        detect new sequences only when there are multiple sequences in the file.


bookMarkDict = json.loads(bookmark)<EOL>realpath = os.path.realpath(self._filename)<EOL>bookMarkFile = bookMarkDict.get('<STR_LIT>', None)<EOL>if bookMarkFile != realpath:<EOL><INDENT>print ("<STR_LIT>"<EOL>"<STR_LIT>") % (<EOL>realpath, bookMarkDict)<EOL>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>return bookMarkDict['<STR_LIT>']<EOL><DEDENT>

Extracts start row from the bookmark information


<EOL>if self._mode == self._FILE_WRITE_MODE:<EOL><INDENT>self._file.flush()<EOL><DEDENT>return sum(<NUM_LIT:1> for line in open(self._filename, self._FILE_READ_MODE))<EOL>

Returns:  count of ALL lines in dataset, including header lines


return self._recordCount<EOL>

:returns: (int) the index of the record that will be read next from
          :meth:`~.FileRecordStream.getNextRecord`.


numLines = self._getTotalLineCount()<EOL>if numLines == <NUM_LIT:0>:<EOL><INDENT>assert self._mode == self._FILE_WRITE_MODE and self._recordCount == <NUM_LIT:0><EOL>numDataRows = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>numDataRows = numLines - self._NUM_HEADER_ROWS<EOL><DEDENT>assert numDataRows >= <NUM_LIT:0><EOL>return numDataRows<EOL>

:returns: (int) count of data rows in dataset (excluding header lines)


if self._file is not None:<EOL><INDENT>self._file.flush()<EOL><DEDENT>


return self<EOL>

Context guard - enter

        Just return the object


def __exit__(self, yupe, value, traceback):

self.close()<EOL>

Context guard - exit

        Ensures that the file is always closed at the end of the 'with' block.
        Lets exceptions propagate.


return self<EOL>

Support for the iterator protocol. Return itself


record = self.getNextRecord()<EOL>if record is None:<EOL><INDENT>raise StopIteration<EOL><DEDENT>return record<EOL>


def __init__(self, origField, deltaField):

self.origField = origField<EOL>self.deltaField = deltaField<EOL>self.previousValue = None<EOL>self.rememberReset = False<EOL>


def initFilter(input, filterInfo = None):

if filterInfo is None:<EOL><INDENT>return None<EOL><DEDENT>filterList = []<EOL>for i, fieldName in enumerate(input.getFieldNames()):<EOL><INDENT>fieldFilter = filterInfo.get(fieldName, None)<EOL>if fieldFilter == None:<EOL><INDENT>continue<EOL><DEDENT>var = dict()<EOL>var['<STR_LIT>'] = None<EOL>min = fieldFilter.get('<STR_LIT>', None)<EOL>max = fieldFilter.get('<STR_LIT>', None)<EOL>var['<STR_LIT>'] = min<EOL>var['<STR_LIT>'] = max<EOL>if fieldFilter['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>var['<STR_LIT>'] = fieldFilter['<STR_LIT>']<EOL>fp = lambda x: (x['<STR_LIT:value>'] != SENTINEL_VALUE_FOR_MISSING_DATA andx['<STR_LIT:value>'] in x['<STR_LIT>'])<EOL><DEDENT>elif fieldFilter['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>if min != None and max != None:<EOL><INDENT>fp = lambda x: (x['<STR_LIT:value>'] != SENTINEL_VALUE_FOR_MISSING_DATA andx['<STR_LIT:value>'] >= x['<STR_LIT>'] and x['<STR_LIT:value>'] <= x['<STR_LIT>'])<EOL><DEDENT>elif min != None:<EOL><INDENT>fp = lambda x: (x['<STR_LIT:value>'] != SENTINEL_VALUE_FOR_MISSING_DATA andx['<STR_LIT:value>'] >= x['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>fp = lambda x: (x['<STR_LIT:value>'] != SENTINEL_VALUE_FOR_MISSING_DATA andx['<STR_LIT:value>'] <= x['<STR_LIT>'])<EOL><DEDENT><DEDENT>filterList.append((i, fp, var))<EOL><DEDENT>return (_filterRecord, filterList)<EOL>

Initializes internal filter variables for further processing.
    Returns a tuple (function to call,parameters for the filter call)

    The filterInfo is a dict. Here is an example structure:
      {fieldName: {'min': x,
                   'max': y,
                   'type': 'category', # or 'number'
                   'acceptValues': ['foo', 'bar'],
                   }
      }

    This returns the following:
      (filterFunc, ((fieldIdx, fieldFilterFunc, filterDict),
                    ...)

    Where fieldIdx is the index of the field within each record
          fieldFilterFunc returns True if the value is "OK" (within min, max or
             part of acceptValues)
          fieldDict is a dict containing 'type', 'min', max', 'acceptValues'


for (fieldIdx, fp, params) in filterList:<EOL><INDENT>x = dict()<EOL>x['<STR_LIT:value>'] = record[fieldIdx]<EOL>x['<STR_LIT>'] = params['<STR_LIT>']<EOL>x['<STR_LIT>'] = params['<STR_LIT>']<EOL>x['<STR_LIT>'] = params['<STR_LIT>']<EOL>if not fp(x):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Takes a record and returns true if record meets filter criteria,
    false otherwise


for elem in inList:<EOL><INDENT>if elem != SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return elem<EOL><DEDENT><DEDENT>return None<EOL>

Returns first non-None element in the list, or None if all are None


for elem in reversed(inList):<EOL><INDENT>if elem != SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>return elem<EOL><DEDENT><DEDENT>return None<EOL>

Returns last non-None element in the list, or None if all are None


aggrMean = _aggr_mean(inList)<EOL>if aggrMean == None:<EOL><INDENT>return None<EOL><DEDENT>aggrSum = <NUM_LIT:0><EOL>for elem in inList:<EOL><INDENT>if elem != SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>aggrSum += elem<EOL><DEDENT>else:<EOL><INDENT>aggrSum += aggrMean<EOL><DEDENT><DEDENT>return aggrSum<EOL>

Returns sum of the elements in the list. Missing items are replaced with
    the mean value


aggrSum = <NUM_LIT:0><EOL>nonNone = <NUM_LIT:0><EOL>for elem in inList:<EOL><INDENT>if elem != SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>aggrSum += elem<EOL>nonNone += <NUM_LIT:1><EOL><DEDENT><DEDENT>if nonNone != <NUM_LIT:0>:<EOL><INDENT>return aggrSum / nonNone<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Returns mean of non-None elements of the list


valueCounts = dict()<EOL>nonNone = <NUM_LIT:0><EOL>for elem in inList:<EOL><INDENT>if elem == SENTINEL_VALUE_FOR_MISSING_DATA:<EOL><INDENT>continue<EOL><DEDENT>nonNone += <NUM_LIT:1><EOL>if elem in valueCounts:<EOL><INDENT>valueCounts[elem] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>valueCounts[elem] = <NUM_LIT:1><EOL><DEDENT><DEDENT>if nonNone == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>sortedCounts = list(valueCounts.items())<EOL>sortedCounts.sort(cmp=lambda x,y: x[<NUM_LIT:1>] - y[<NUM_LIT:1>], reverse=True)<EOL>return sortedCounts[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>

Returns most common value seen in the non-None elements of the list


assert(len(inList) == len(params))<EOL>weightsSum = sum(params)<EOL>if weightsSum == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>weightedMean = <NUM_LIT:0><EOL>for i, elem in enumerate(inList):<EOL><INDENT>weightedMean += elem * params[i]<EOL><DEDENT>return weightedMean / weightsSum<EOL>

Weighted mean uses params (must be the same size as inList) and
    makes weighed mean of inList


def generateDataset(aggregationInfo, inputFilename, outputFilename=None):

<EOL>inputFullPath = resource_filename("<STR_LIT>", inputFilename)<EOL>inputObj = FileRecordStream(inputFullPath)<EOL>aggregator = Aggregator(aggregationInfo=aggregationInfo,<EOL>inputFields=inputObj.getFields())<EOL>if aggregator.isNullAggregation():<EOL><INDENT>return inputFullPath<EOL><DEDENT>if outputFilename is None:<EOL><INDENT>outputFilename = '<STR_LIT>' %os.path.splitext(os.path.basename(inputFullPath))[<NUM_LIT:0>]<EOL>timePeriods = '<STR_LIT>''<STR_LIT>'<EOL>for k in timePeriods.split():<EOL><INDENT>if aggregationInfo.get(k, <NUM_LIT:0>) > <NUM_LIT:0>:<EOL><INDENT>outputFilename += '<STR_LIT>' % (k, aggregationInfo[k])<EOL><DEDENT><DEDENT>outputFilename += '<STR_LIT>'<EOL>outputFilename = os.path.join(os.path.dirname(inputFullPath), outputFilename)<EOL><DEDENT>lockFilePath = outputFilename + '<STR_LIT>'<EOL>if os.path.isfile(outputFilename) oros.path.isfile(lockFilePath):<EOL><INDENT>while os.path.isfile(lockFilePath):<EOL><INDENT>print('<STR_LIT>' %lockFilePath)<EOL>time.sleep(<NUM_LIT:1>)<EOL><DEDENT>return outputFilename<EOL><DEDENT>lockFD = open(lockFilePath, '<STR_LIT:w>')<EOL>outputObj = FileRecordStream(streamID=outputFilename, write=True,<EOL>fields=inputObj.getFields())<EOL>while True:<EOL><INDENT>inRecord = inputObj.getNextRecord()<EOL>(aggRecord, aggBookmark) = aggregator.next(inRecord, None)<EOL>if aggRecord is None and inRecord is None:<EOL><INDENT>break<EOL><DEDENT>if aggRecord is not None:<EOL><INDENT>outputObj.appendRecord(aggRecord)<EOL><DEDENT><DEDENT>return outputFilename<EOL>

Generate a dataset of aggregated values

    Parameters:
    ----------------------------------------------------------------------------
    aggregationInfo: a dictionary that contains the following entries
      - fields: a list of pairs. Each pair is a field name and an
        aggregation function (e.g. sum). The function will be used to aggregate
        multiple values during the aggregation period.

    aggregation period: 0 or more of unit=value fields; allowed units are:
          [years months] |
          [weeks days hours minutes seconds milliseconds microseconds]
          NOTE: years and months are mutually-exclusive with the other units.
                See getEndTime() and _aggregate() for more details.
          Example1: years=1, months=6,
          Example2: hours=1, minutes=30,
          If none of the period fields are specified or if all that are specified
          have values of 0, then aggregation will be suppressed, and the given
          inputFile parameter value will be returned.

    inputFilename: filename of the input dataset within examples/prediction/data

    outputFilename: name for the output file. If not given, a name will be
          generated based on the input filename and the aggregation params

    retval: Name of the generated output file. This will be the same as the input
        file name if no aggregation needed to be performed

    If the input file contained a time field, sequence id field or reset field
    that were not specified in aggregationInfo fields, those fields will be
    added automatically with the following rules:

    1. The order will be R, S, T, rest of the fields
    2. The aggregation function for all will be to pick the first: lambda x: x[0]

      Returns: the path of the aggregated data file if aggregation was performed
        (in the same directory as the given input file); if aggregation did not
        need to be performed, then the given inputFile argument value is returned.


<EOL>inputFile = resource_filename("<STR_LIT>", inputFile)<EOL>a = defaultdict(lambda: <NUM_LIT:0>, aggregationInfo)<EOL>outputDir = os.path.dirname(inputFile)<EOL>outputFile = '<STR_LIT>' % os.path.splitext(os.path.basename(inputFile))[<NUM_LIT:0>]<EOL>noAggregation = True<EOL>timePeriods = '<STR_LIT>''<STR_LIT>'<EOL>for k in timePeriods.split():<EOL><INDENT>if a[k] > <NUM_LIT:0>:<EOL><INDENT>noAggregation = False<EOL>outputFile += '<STR_LIT>' % (k, a[k])<EOL><DEDENT><DEDENT>if noAggregation:<EOL><INDENT>return inputFile<EOL><DEDENT>outputFile += '<STR_LIT>'<EOL>outputFile = os.path.join(outputDir, outputFile)<EOL>return outputFile<EOL>

Generate the filename for aggregated dataset

    The filename is based on the input filename and the
    aggregation period.

    Returns the inputFile if no aggregation required (aggregation
    info has all 0's)


def __init__(self, aggregationInfo, inputFields, timeFieldName=None,<EOL>sequenceIdFieldName=None, resetFieldName=None, filterInfo=None):

<EOL>self._filterInfo = filterInfo<EOL>self._nullAggregation = False<EOL>self._inputFields = inputFields<EOL>self._nullAggregation = False<EOL>if aggregationInfo is None:<EOL><INDENT>self._nullAggregation = True<EOL><DEDENT>else:<EOL><INDENT>aggDef = defaultdict(lambda: <NUM_LIT:0>, aggregationInfo)<EOL>if (aggDef['<STR_LIT>'] == aggDef['<STR_LIT>'] == aggDef['<STR_LIT>'] ==<EOL>aggDef['<STR_LIT>'] == aggDef['<STR_LIT>'] == aggDef['<STR_LIT>'] ==<EOL>aggDef['<STR_LIT>'] == aggDef['<STR_LIT>'] ==<EOL>aggDef['<STR_LIT>'] == <NUM_LIT:0>):<EOL><INDENT>self._nullAggregation = True<EOL><DEDENT><DEDENT>self._filter = initFilter(self._inputFields, self._filterInfo)<EOL>self._fields = None<EOL>self._resetFieldIdx = None<EOL>self._timeFieldIdx = None<EOL>self._sequenceIdFieldIdx = None<EOL>self._aggTimeDelta = datetime.timedelta()<EOL>self._aggYears = <NUM_LIT:0><EOL>self._aggMonths = <NUM_LIT:0><EOL>self._aggrInputBookmark = None<EOL>self._startTime = None<EOL>self._endTime = None<EOL>self._sequenceId = None<EOL>self._firstSequenceStartTime = None<EOL>self._inIdx = -<NUM_LIT:1><EOL>self._slice = defaultdict(list)<EOL>if not self._nullAggregation:<EOL><INDENT>fieldNames = [f[<NUM_LIT:0>] for f in aggregationInfo['<STR_LIT>']]<EOL>readerFieldNames = [f[<NUM_LIT:0>] for f in self._inputFields]<EOL>for name in fieldNames:<EOL><INDENT>if not name in readerFieldNames:<EOL><INDENT>raise Exception('<STR_LIT>' % (name))<EOL><DEDENT><DEDENT>if timeFieldName is not None:<EOL><INDENT>self._timeFieldIdx = readerFieldNames.index(timeFieldName)<EOL><DEDENT>if resetFieldName is not None:<EOL><INDENT>self._resetFieldIdx = readerFieldNames.index(resetFieldName)<EOL><DEDENT>if sequenceIdFieldName is not None:<EOL><INDENT>self._sequenceIdFieldIdx = readerFieldNames.index(sequenceIdFieldName)<EOL><DEDENT>self._fields = []<EOL>fieldIdx = -<NUM_LIT:1><EOL>for (name, type, special) in self._inputFields:<EOL><INDENT>fieldIdx += <NUM_LIT:1><EOL>found = False<EOL>for field in aggregationInfo['<STR_LIT>']:<EOL><INDENT>if field[<NUM_LIT:0>] == name:<EOL><INDENT>aggFunctionName = field[<NUM_LIT:1>]<EOL>found = True<EOL>break<EOL><DEDENT><DEDENT>if not found:<EOL><INDENT>aggFunctionName = '<STR_LIT>'<EOL><DEDENT>(funcPtr, params) = self._getFuncPtrAndParams(aggFunctionName)<EOL>self._fields.append((fieldIdx, funcPtr, params))<EOL>if special == FieldMetaSpecial.reset and self._resetFieldIdx is None:<EOL><INDENT>self._resetFieldIdx = fieldIdx<EOL><DEDENT>if special == FieldMetaSpecial.timestamp and self._timeFieldIdx is None:<EOL><INDENT>self._timeFieldIdx = fieldIdx<EOL><DEDENT>if (special == FieldMetaSpecial.sequence and<EOL>self._sequenceIdFieldIdx is None):<EOL><INDENT>self._sequenceIdFieldIdx = fieldIdx<EOL><DEDENT><DEDENT>assert self._timeFieldIdx is not None, "<STR_LIT>"<EOL>self._aggTimeDelta = datetime.timedelta(days=aggDef['<STR_LIT>'],<EOL>hours=aggDef['<STR_LIT>'],<EOL>minutes=aggDef['<STR_LIT>'],<EOL>seconds=aggDef['<STR_LIT>'],<EOL>milliseconds=aggDef['<STR_LIT>'],<EOL>microseconds=aggDef['<STR_LIT>'],<EOL>weeks=aggDef['<STR_LIT>'])<EOL>self._aggYears = aggDef['<STR_LIT>']<EOL>self._aggMonths = aggDef['<STR_LIT>']<EOL>if self._aggTimeDelta:<EOL><INDENT>assert self._aggYears == <NUM_LIT:0><EOL>assert self._aggMonths == <NUM_LIT:0><EOL><DEDENT><DEDENT>

Construct an aggregator instance

        Params:

        - aggregationInfo: a dictionary that contains the following entries
          - fields: a list of pairs. Each pair is a field name and an
            aggregation function (e.g. sum). The function will be used to aggregate
            multiple values during the aggregation period.

          - aggregation period: 0 or more of unit=value fields; allowed units are:
              [years months] | [weeks days hours minutes seconds milliseconds
              microseconds]
              NOTE: years and months are mutually-exclusive with the other units.  See
                    getEndTime() and _aggregate() for more details.
              Example1: years=1, months=6,
              Example2: hours=1, minutes=30,
              If none of the period fields are specified or if all that are specified
              have values of 0, then aggregation will be suppressed, and the given
              inputFile parameter value will be returned.

        - inputFields: The fields from the data source. This is a sequence of
          `nupic.data.fieldmeta.FieldMetaInfo` instances.

        - timeFieldName: name of the field to use as the time field. If None,
              then the time field will be queried from the reader.

        - sequenceIdFieldName: name of the field to use as the sequenecId. If None,
              then the time field will be queried from the reader.

        - resetFieldName: name of the field to use as the reset field. If None,
              then the time field will be queried from the reader.

        - filterInfo: a structure with rules for filtering records out

        If the input file contains a time field, sequence id field or reset field
        that were not specified in aggregationInfo fields, those fields will be
        added automatically with the following rules:

        1. The order will be R, S, T, rest of the fields
        2. The aggregation function for these will be to pick the first:
           lambda x: x[0]


assert isinstance(t, datetime.datetime)<EOL>if self._aggTimeDelta:<EOL><INDENT>return t + self._aggTimeDelta<EOL><DEDENT>else:<EOL><INDENT>year = t.year + self._aggYears + (t.month - <NUM_LIT:1> + self._aggMonths) / <NUM_LIT:12><EOL>month = (t.month - <NUM_LIT:1> + self._aggMonths) % <NUM_LIT:12> + <NUM_LIT:1><EOL>return t.replace(year=year, month=month)<EOL><DEDENT>

Add the aggregation period to the input time t and return a datetime object

        Years and months are handled as aspecial case due to leap years
        and months with different number of dates. They can't be converted
        to a strict timedelta because a period of 3 months will have different
        durations actually. The solution is to just add the years and months
        fields directly to the current time.

        Other periods are converted to timedelta and just added to current time.


params = None<EOL>if isinstance(funcName, str):<EOL><INDENT>if funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_sum<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_first<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_last<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_mean<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = max<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = min<EOL><DEDENT>elif funcName == '<STR_LIT>':<EOL><INDENT>fp = _aggr_mode<EOL><DEDENT>elif funcName.startswith('<STR_LIT>'):<EOL><INDENT>fp = _aggr_weighted_mean<EOL>paramsName = funcName[<NUM_LIT:6>:]<EOL>params = [f[<NUM_LIT:0>] for f in self._inputFields].index(paramsName)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fp = funcName<EOL><DEDENT>return (fp, params)<EOL>

Given the name of an aggregation function, returns the function pointer
        and param.

        Parameters:
        ------------------------------------------------------------------------
        funcName:  a string (name of function) or funcPtr
        retval:   (funcPtr, param)


record = []<EOL>for i, (fieldIdx, aggFP, paramIdx) in enumerate(self._fields):<EOL><INDENT>if aggFP is None: <EOL><INDENT>continue<EOL><DEDENT>values = self._slice[i]<EOL>refIndex = None<EOL>if paramIdx is not None:<EOL><INDENT>record.append(aggFP(values, self._slice[paramIdx]))<EOL><DEDENT>else:<EOL><INDENT>record.append(aggFP(values))<EOL><DEDENT><DEDENT>return record<EOL>


return self._nullAggregation<EOL>

Return True if no aggregation will be performed, either because the
        aggregationInfo was None or all aggregation params within it were 0.


def next(self, record, curInputBookmark):

<EOL>outRecord = None<EOL>retInputBookmark = None<EOL>if record is not None:<EOL><INDENT>self._inIdx += <NUM_LIT:1><EOL>if self._filter != None and not self._filter[<NUM_LIT:0>](self._filter[<NUM_LIT:1>], record):<EOL><INDENT>return (None, None)<EOL><DEDENT>if self._nullAggregation:<EOL><INDENT>return (record, curInputBookmark)<EOL><DEDENT>t = record[self._timeFieldIdx]<EOL>if self._firstSequenceStartTime == None:<EOL><INDENT>self._firstSequenceStartTime = t<EOL><DEDENT>if self._startTime is None:<EOL><INDENT>self._startTime = t<EOL><DEDENT>if self._endTime is None:<EOL><INDENT>self._endTime = self._getEndTime(t)<EOL>assert self._endTime > t<EOL><DEDENT>if self._resetFieldIdx is not None:<EOL><INDENT>resetSignal = record[self._resetFieldIdx]<EOL><DEDENT>else:<EOL><INDENT>resetSignal = None<EOL><DEDENT>if self._sequenceIdFieldIdx is not None:<EOL><INDENT>currSequenceId = record[self._sequenceIdFieldIdx]<EOL><DEDENT>else:<EOL><INDENT>currSequenceId = None<EOL><DEDENT>newSequence = (resetSignal == <NUM_LIT:1> and self._inIdx > <NUM_LIT:0>)or self._sequenceId != currSequenceIdor self._inIdx == <NUM_LIT:0><EOL>if newSequence:<EOL><INDENT>self._sequenceId = currSequenceId<EOL><DEDENT>sliceEnded = (t >= self._endTime or t < self._startTime)<EOL>if (newSequence or sliceEnded) and len(self._slice) > <NUM_LIT:0>:<EOL><INDENT>for j, f in enumerate(self._fields):<EOL><INDENT>index = f[<NUM_LIT:0>]<EOL>if index == self._timeFieldIdx:<EOL><INDENT>self._slice[j][<NUM_LIT:0>] = self._startTime<EOL>break<EOL><DEDENT><DEDENT>outRecord = self._createAggregateRecord()<EOL>retInputBookmark = self._aggrInputBookmark<EOL>self._slice = defaultdict(list)<EOL><DEDENT>for j, f in enumerate(self._fields):<EOL><INDENT>index = f[<NUM_LIT:0>]<EOL>self._slice[j].append(record[index])<EOL>self._aggrInputBookmark = curInputBookmark<EOL><DEDENT>if newSequence:<EOL><INDENT>self._startTime = t<EOL>self._endTime = self._getEndTime(t)<EOL><DEDENT>if sliceEnded:<EOL><INDENT>if t < self._startTime:<EOL><INDENT>self._endTime = self._firstSequenceStartTime<EOL><DEDENT>while t >= self._endTime:<EOL><INDENT>self._startTime = self._endTime<EOL>self._endTime = self._getEndTime(self._endTime)<EOL><DEDENT><DEDENT>if outRecord is not None:<EOL><INDENT>return (outRecord, retInputBookmark)<EOL><DEDENT><DEDENT>elif self._slice:<EOL><INDENT>for j, f in enumerate(self._fields):<EOL><INDENT>index = f[<NUM_LIT:0>]<EOL>if index == self._timeFieldIdx:<EOL><INDENT>self._slice[j][<NUM_LIT:0>] = self._startTime<EOL>break<EOL><DEDENT><DEDENT>outRecord = self._createAggregateRecord()<EOL>retInputBookmark = self._aggrInputBookmark<EOL>self._slice = defaultdict(list)<EOL><DEDENT>return (outRecord, retInputBookmark)<EOL>

Return the next aggregated record, if any

        Parameters:
        ------------------------------------------------------------------------
        record:         The input record (values only) from the input source, or
                        None if the input has reached EOF (this will cause this
                        method to force completion of and return any partially
                        aggregated time period)
        curInputBookmark: The bookmark to the next input record
        retval:
          (outputRecord, inputBookmark)

          outputRecord: the aggregated record
          inputBookmark: a bookmark to the last position from the input that
                          contributed to this aggregated record.

          If we don't have any aggregated records yet, returns (None, None)

        The caller should generally do a loop like this:
          while True:
            inRecord = reader.getNextRecord()
            bookmark = reader.getBookmark()

            (aggRecord, aggBookmark) = aggregator.next(inRecord, bookmark)

            # reached EOF?
            if inRecord is None and aggRecord is None:
              break

            if aggRecord is not None:
              proessRecord(aggRecord, aggBookmark)

        This method makes use of the self._slice member variable to build up
        the values we need to aggregate. This is a dict of lists. The keys are
        the field indices and the elements of each list are the values for that
        field. For example:

          self._siice = { 0: [42, 53], 1: [4.0, 5.1] }


for i, field in enumerate(fields):<EOL><INDENT>if field.special == special:<EOL><INDENT>return i<EOL><DEDENT><DEDENT>return None<EOL>

Return index of the field matching the field meta special value.
    :param fields: sequence of nupic.data.fieldmeta.FieldMetaInfo objects
      representing the fields of a stream
    :param special: one of the special field attribute values from
      nupic.data.fieldmeta.FieldMetaSpecial
    :returns: first zero-based index of the field tagged with the target field
      meta special attribute; None if no such field


def __init__(self, fields, aggregationPeriod=None):

if not fields:<EOL><INDENT>raise ValueError('<STR_LIT>' % (fields,))<EOL><DEDENT>self._fields = fields<EOL>self._aggregationPeriod = aggregationPeriod<EOL>self._sequenceId = -<NUM_LIT:1><EOL>self._fieldNames = tuple(f.name for f in fields)<EOL>self._categoryFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.category)<EOL>self._resetFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.reset)<EOL>self._sequenceFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.sequence)<EOL>self._timestampFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.timestamp)<EOL>self._learningFieldIndex = _getFieldIndexBySpecial(<EOL>fields,<EOL>FieldMetaSpecial.learning)<EOL>

:param fields: non-empty sequence of nupic.data.fieldmeta.FieldMetaInfo
  objects corresponding to fields in input rows.
:param aggregationPeriod: (dict) aggregation period of the record stream
  containing 'months' and 'seconds'. The months is always an integer
  and seconds is a floating point. Only one is allowed to be non-zero at a
  time. If there is no aggregation associated with the stream, pass None.
  Typically, a raw file or hbase stream will NOT have any aggregation info,
  but subclasses of RecordStreamIface, like StreamReader, will and will
  provide the aggregation period. This is used by the encode method to
  assign a record number to a record given its timestamp and the aggregation
  interval.


self._sequenceId = -<NUM_LIT:1><EOL>

Put us back at the beginning of the file again


<EOL>result = dict(zip(self._fieldNames, inputRow))<EOL>if self._categoryFieldIndex is not None:<EOL><INDENT>if isinstance(inputRow[self._categoryFieldIndex], int):<EOL><INDENT>result['<STR_LIT>'] = [inputRow[self._categoryFieldIndex]]<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = (inputRow[self._categoryFieldIndex]<EOL>if inputRow[self._categoryFieldIndex]<EOL>else [None])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = [None]<EOL><DEDENT>if self._resetFieldIndex is not None:<EOL><INDENT>result['<STR_LIT>'] = int(bool(inputRow[self._resetFieldIndex]))<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = <NUM_LIT:0><EOL><DEDENT>if self._learningFieldIndex is not None:<EOL><INDENT>result['<STR_LIT>'] = int(bool(inputRow[self._learningFieldIndex]))<EOL><DEDENT>result['<STR_LIT>'] = None<EOL>if self._timestampFieldIndex is not None:<EOL><INDENT>result['<STR_LIT>'] = inputRow[self._timestampFieldIndex]<EOL>result['<STR_LIT>'] = self._computeTimestampRecordIdx(<EOL>inputRow[self._timestampFieldIndex])<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = None<EOL><DEDENT>hasReset = self._resetFieldIndex is not None<EOL>hasSequenceId = self._sequenceFieldIndex is not None<EOL>if hasReset and not hasSequenceId:<EOL><INDENT>if result['<STR_LIT>']:<EOL><INDENT>self._sequenceId += <NUM_LIT:1><EOL><DEDENT>sequenceId = self._sequenceId<EOL><DEDENT>elif not hasReset and hasSequenceId:<EOL><INDENT>sequenceId = inputRow[self._sequenceFieldIndex]<EOL>result['<STR_LIT>'] = int(sequenceId != self._sequenceId)<EOL>self._sequenceId = sequenceId<EOL><DEDENT>elif hasReset and hasSequenceId:<EOL><INDENT>sequenceId = inputRow[self._sequenceFieldIndex]<EOL><DEDENT>else:<EOL><INDENT>sequenceId = <NUM_LIT:0><EOL><DEDENT>if sequenceId is not None:<EOL><INDENT>result['<STR_LIT>'] = hash(sequenceId)<EOL><DEDENT>else:<EOL><INDENT>result['<STR_LIT>'] = None<EOL><DEDENT>return result<EOL>

Encodes the given input row as a dict, with the
        keys being the field names. This also adds in some meta fields:
          '_category': The value from the category field (if any)
          '_reset': True if the reset field was True (if any)
          '_sequenceId': the value from the sequenceId field (if any)

        :param inputRow: sequence of values corresponding to a single input metric
          data row
        :rtype: dict


if self._aggregationPeriod is None:<EOL><INDENT>return None<EOL><DEDENT>if self._aggregationPeriod['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>assert self._aggregationPeriod['<STR_LIT>'] == <NUM_LIT:0><EOL>result = int(<EOL>(recordTS.year * <NUM_LIT:12> + (recordTS.month-<NUM_LIT:1>)) /<EOL>self._aggregationPeriod['<STR_LIT>'])<EOL><DEDENT>elif self._aggregationPeriod['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>delta = recordTS - datetime.datetime(year=<NUM_LIT:1>, month=<NUM_LIT:1>, day=<NUM_LIT:1>)<EOL>deltaSecs = delta.days * <NUM_LIT> * <NUM_LIT> * <NUM_LIT>+ delta.seconds+ delta.microseconds / <NUM_LIT><EOL>result = int(deltaSecs / self._aggregationPeriod['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>result = None<EOL><DEDENT>return result<EOL>

Give the timestamp of a record (a datetime object), compute the record's
        timestamp index - this is the timestamp divided by the aggregation period.

        Parameters:
        ------------------------------------------------------------------------
        recordTS:  datetime instance
        retval:    record timestamp index, or None if no aggregation period


if self._modelRecordEncoder is not None:<EOL><INDENT>self._modelRecordEncoder.rewind()<EOL><DEDENT>

Put us back at the beginning of the file again.


Returns next available data record from the storage. If ``useCache`` is
``False``, then don't read ahead and don't cache any records.

:return: a data row (a list or tuple) if available; None, if no more records
         in the table (End of Stream - EOS); empty sequence (list or tuple)
         when timing out while waiting for the next record.


values = self.getNextRecord()<EOL>if values is None:<EOL><INDENT>return None<EOL><DEDENT>if not values:<EOL><INDENT>return dict()<EOL><DEDENT>if self._modelRecordEncoder is None:<EOL><INDENT>self._modelRecordEncoder = ModelRecordEncoder(<EOL>fields=self.getFields(),<EOL>aggregationPeriod=self.getAggregationMonthsAndSeconds())<EOL><DEDENT>return self._modelRecordEncoder.encode(values)<EOL>

Returns next available data record from the storage as a dict, with the
        keys being the field names. This also adds in some meta fields:

          - ``_category``: The value from the category field (if any)
          - ``_reset``: True if the reset field was True (if any)
          - ``_sequenceId``: the value from the sequenceId field (if any)


return None<EOL>

Returns the aggregation period of the record stream as a dict
containing 'months' and 'seconds'. The months is always an integer and
seconds is a floating point. Only one is allowed to be non-zero.

If there is no aggregation associated with the stream, returns None.

Typically, a raw file or hbase stream will NOT have any aggregation info,
but subclasses of :class:`~nupic.data.record_stream.RecordStreamIface`, like
:class:`~nupic.data.stream_reader.StreamReader`, will and will return the
aggregation period from this call. This call is used by
:meth:`getNextRecordDict` to assign a record number to a record given its
timestamp and the aggregation interval.

:returns: ``None``


:returns: (int) index of the record that will be read next from
          :meth:`getNextRecord`


Saves the record in the underlying storage. Should be implemented in
subclasses.

:param record: (object) to store


@abstractmethod<EOL><INDENT>def appendRecords(self, records, progressCB=None):<DEDENT>

Saves multiple records in the underlying storage. Should be implemented in
subclasses.

:param records: (list) of objects to store
:param progressCB: (func) called after each appension


Returns an anchor to the current position in the data. Passing this
        anchor to the constructor makes the current position to be the first
        returned record. If record is no longer in the storage, the first available
        after it will be returned.

        :returns: anchor to current position in the data.


:param bookmark: (int) where to start
:returns: True if there are records left after the  bookmark.


:param numRecords: (int) number of records from the end.
:returns: (int) a bookmark numRecords from the end of the stream.


:returns: storage stats (like min and max values of the fields).


stats = self.getStats()<EOL>if stats == None:<EOL><INDENT>return None<EOL><DEDENT>minValues = stats.get('<STR_LIT>', None)<EOL>if minValues == None:<EOL><INDENT>return None<EOL><DEDENT>index = self.getFieldNames().index(fieldName)<EOL>return minValues[index]<EOL>

If underlying implementation does not support min/max stats collection,
or if a field type does not support min/max (non scalars), the return
value will be None.

:param fieldName: (string) name of field to get min
:returns: current minimum value for the field ``fieldName``.


stats = self.getStats()<EOL>if stats == None:<EOL><INDENT>return None<EOL><DEDENT>maxValues = stats.get('<STR_LIT>', None)<EOL>if maxValues == None:<EOL><INDENT>return None<EOL><DEDENT>index = self.getFieldNames().index(fieldName)<EOL>return maxValues[index]<EOL>

If underlying implementation does not support min/max stats collection,
or if a field type does not support min/max (non scalars), the return
value will be None.

:param fieldName: (string) name of field to get max
:returns: current maximum value for the field ``fieldName``.


Saves specified error in the storage.

:param error: Error to store.


:returns: True if all records are already in the storage or False
          if more records is expected.


Marks the stream completed.

:param completed: (bool) is completed?


:returns: (list) of field names associated with the data.


:returns: (list) of :class:`nupic.data.fieldmeta.FieldMetaInfo` objects for
    each field in the stream. Might be None, if that information is provided
    externally (through the `Stream Definition <stream-def.html>`_,
    for example).


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.reset)<EOL>

:returns: (int) index of the ``reset`` field; ``None`` if no such field.


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.timestamp)<EOL>

:returns: (int) index of the ``timestamp`` field.


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.sequence)<EOL>

:returns: (int) index of the ``sequenceId`` field.


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.category)<EOL>

:returns: (int) index of ``category`` field


return _getFieldIndexBySpecial(self.getFields(), FieldMetaSpecial.learning)<EOL>

:returns: (int) index of the ``learning`` field.


Set the read timeout in seconds

:param timeout: (int or floating point)


def generateStats(filename, statsInfo, maxSamples = None, filters=[], cache=True):

<EOL>if not isinstance(statsInfo, dict):<EOL><INDENT>raise RuntimeError("<STR_LIT>"<EOL>"<STR_LIT>" % type(statsInfo))<EOL><DEDENT>filename = resource_filename("<STR_LIT>", filename)<EOL>if cache:<EOL><INDENT>statsFilename = getStatsFilename(filename, statsInfo, filters)<EOL>if os.path.exists(statsFilename):<EOL><INDENT>try:<EOL><INDENT>r = pickle.load(open(statsFilename, "<STR_LIT:rb>"))<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>""<STR_LIT>" % filename)<EOL>r = dict()<EOL><DEDENT>requestedKeys = set([s for s in statsInfo])<EOL>availableKeys = set(r.keys())<EOL>unavailableKeys = requestedKeys.difference(availableKeys)<EOL>if len(unavailableKeys ) == <NUM_LIT:0>:<EOL><INDENT>return r<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>""<STR_LIT>" %(filename, str(unavailableKeys)))<EOL>os.remove(filename)<EOL><DEDENT><DEDENT><DEDENT>print("<STR_LIT>" % (filename, filters))<EOL>sensor = RecordSensor()<EOL>sensor.dataSource = FileRecordStream(filename)<EOL>sensor.preEncodingFilters = filters<EOL>stats = []<EOL>for field in statsInfo:<EOL><INDENT>if statsInfo[field] == "<STR_LIT>":<EOL><INDENT>statsInfo[field] = NumberStatsCollector()<EOL><DEDENT>elif statsInfo[field] == "<STR_LIT>":<EOL><INDENT>statsInfo[field] = CategoryStatsCollector()<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % (statsInfo[field], field))<EOL><DEDENT><DEDENT>if maxSamples is None:<EOL><INDENT>maxSamples = <NUM_LIT><EOL><DEDENT>for i in range(maxSamples):<EOL><INDENT>try:<EOL><INDENT>record = sensor.getNextRecord()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>for (name, collector) in list(statsInfo.items()):<EOL><INDENT>collector.add(record[name])<EOL><DEDENT><DEDENT>del sensor<EOL>r = dict()<EOL>for (field, collector) in list(statsInfo.items()):<EOL><INDENT>stats = collector.getStats()<EOL>if field not in r:<EOL><INDENT>r[field] = stats<EOL><DEDENT>else:<EOL><INDENT>r[field].update(stats)<EOL><DEDENT><DEDENT>if cache:<EOL><INDENT>f = open(statsFilename, "<STR_LIT:wb>")<EOL>pickle.dump(r, f)<EOL>f.close()<EOL>r["<STR_LIT>"] = statsFilename<EOL><DEDENT>return r<EOL>

Generate requested statistics for a dataset and cache to a file.
    If filename is None, then don't cache to a file


def generateStats(filename, maxSamples = None,):

<EOL>statsCollectorMapping = {'<STR_LIT:float>':    FloatStatsCollector,<EOL>'<STR_LIT:int>':      IntStatsCollector,<EOL>'<STR_LIT:string>':   StringStatsCollector,<EOL>'<STR_LIT>': DateTimeStatsCollector,<EOL>'<STR_LIT:bool>':     BoolStatsCollector,<EOL>}<EOL>filename = resource_filename("<STR_LIT>", filename)<EOL>print("<STR_LIT:*>"*<NUM_LIT>)<EOL>print("<STR_LIT>" % (filename,))<EOL>dataFile = FileRecordStream(filename)<EOL>statsCollectors = []<EOL>for fieldName, fieldType, fieldSpecial in dataFile.getFields():<EOL><INDENT>statsCollector =statsCollectorMapping[fieldType](fieldName, fieldType, fieldSpecial)<EOL>statsCollectors.append(statsCollector)<EOL><DEDENT>if maxSamples is None:<EOL><INDENT>maxSamples = <NUM_LIT><EOL><DEDENT>for i in range(maxSamples):<EOL><INDENT>record = dataFile.getNextRecord()<EOL>if record is None:<EOL><INDENT>break<EOL><DEDENT>for i, value in enumerate(record):<EOL><INDENT>statsCollectors[i].addValue(value)<EOL><DEDENT><DEDENT>stats = {}<EOL>for statsCollector in statsCollectors:<EOL><INDENT>statsCollector.getStats(stats)<EOL><DEDENT>if dataFile.getResetFieldIdx() is not None:<EOL><INDENT>resetFieldName,_,_ = dataFile.getFields()[dataFile.reset]<EOL>stats.pop(resetFieldName)<EOL><DEDENT>if VERBOSITY > <NUM_LIT:0>:<EOL><INDENT>pprint.pprint(stats)<EOL><DEDENT>return stats<EOL>

Collect statistics for each of the fields in the user input data file and
return a stats dict object.

Parameters:
------------------------------------------------------------------------------
filename:             The path and name of the data file.
maxSamples:           Upper bound on the number of rows to be processed
retval:               A dictionary of dictionaries. The top level keys are the
                      field names and the corresponding values are the statistics
                      collected for the individual file.
                      Example:
                      {
                        'consumption':{'min':0,'max':90,'mean':50,...},
                        'gym':{'numDistinctCategories':10,...},
                        ...
                       }


BaseStatsCollector.getStats(self, stats)<EOL>sortedNumberList = sorted(self.valueList)<EOL>listLength = len(sortedNumberList)<EOL>min = sortedNumberList[<NUM_LIT:0>]<EOL>max = sortedNumberList[-<NUM_LIT:1>]<EOL>mean = numpy.mean(self.valueList)<EOL>median = sortedNumberList[int(<NUM_LIT:0.5>*listLength)]<EOL>percentile1st = sortedNumberList[int(<NUM_LIT>*listLength)]<EOL>percentile99th = sortedNumberList[int(<NUM_LIT>*listLength)]<EOL>differenceList =[(cur - prev) for prev, cur in zip(list(self.valueSet)[:-<NUM_LIT:1>],<EOL>list(self.valueSet)[<NUM_LIT:1>:])]<EOL>if min > max:<EOL><INDENT>print(self.fieldname, min, max, '<STR_LIT>')<EOL><DEDENT>meanResolution = numpy.mean(differenceList)<EOL>stats[self.fieldname]['<STR_LIT>'] = min<EOL>stats[self.fieldname]['<STR_LIT>'] = max<EOL>stats[self.fieldname]['<STR_LIT>'] = mean<EOL>stats[self.fieldname]['<STR_LIT>'] = median<EOL>stats[self.fieldname]['<STR_LIT>'] = percentile1st<EOL>stats[self.fieldname]['<STR_LIT>'] = percentile99th<EOL>stats[self.fieldname]['<STR_LIT>'] = meanResolution<EOL>passData = True<EOL>if passData:<EOL><INDENT>stats[self.fieldname]['<STR_LIT:data>'] = self.valueList<EOL><DEDENT>if VERBOSITY > <NUM_LIT:2>:<EOL><INDENT>print('<STR_LIT>')<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>", min)<EOL>print("<STR_LIT>", max)<EOL>print("<STR_LIT>", mean)<EOL>print("<STR_LIT>", median)<EOL>print("<STR_LIT>", percentile1st)<EOL>print("<STR_LIT>", percentile99th)<EOL>print('<STR_LIT>')<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>", meanResolution)<EOL><DEDENT>if VERBOSITY > <NUM_LIT:3>:<EOL><INDENT>print('<STR_LIT>')<EOL>print("<STR_LIT>")<EOL>counts, bins = numpy.histogram(self.valueList, new=True)<EOL>print("<STR_LIT>", counts.tolist())<EOL>print("<STR_LIT>", bins.tolist())<EOL><DEDENT>

Override of getStats()  in BaseStatsCollector

            stats: A dictionary where all the stats are
            outputted


self.filterDict = filterDict<EOL>


for field, meta in self.filterDict.iteritems():<EOL><INDENT>index = meta['<STR_LIT:index>']<EOL>categories = meta['<STR_LIT>']<EOL>for category in categories:<EOL><INDENT>if not record:<EOL><INDENT>continue<EOL><DEDENT>if record[index].find(category) != -<NUM_LIT:1>:<EOL><INDENT>'''<STR_LIT>'''<EOL>return True<EOL><DEDENT><DEDENT><DEDENT>return False<EOL>

Returns True if the record matches any of the provided filters


return FieldMetaInfo._make(fieldInfoTuple)<EOL>

Creates a :class:`.field_meta.FieldMetaInfo` instance from a tuple containing
``name``, ``type``, and ``special``.

:param fieldInfoTuple: Must contain ``name``, ``type``, and ``special``
:return: (:class:`~.field_meta.FieldMetaInfo`) instance


return [cls.createFromFileFieldElement(f) for f in fields]<EOL>

Creates a FieldMetaInfo list from the a list of tuples. Basically runs
:meth:`~.field_meta.FieldMetaInfo.createFromFileFieldElement` on each tuple.

*Example:*

.. code-block:: python

    # Create a list of FieldMetaInfo instances from a list of File meta-data
    # tuples
    el = [("pounds", FieldMetaType.float, FieldMetaSpecial.none),
          ("price", FieldMetaType.float, FieldMetaSpecial.none),
          ("id", FieldMetaType.string, FieldMetaSpecial.sequence),
          ("date", FieldMetaType.datetime, FieldMetaSpecial.timestamp),
         ]
    ml = FieldMetaInfo.createListFromFileFieldList(el)

:param fields: a sequence of field attribute tuples conforming to the format
               of ``name``, ``type``, and ``special``

:return: A list of :class:`~.field_meta.FieldMetaInfo` elements corresponding
         to the given 'fields' list.


return fieldDataType in cls._ALL<EOL>

Check a candidate value whether it's one of the valid field data types

        :param fieldDataType: (string) candidate field data type
        :returns: True if the candidate value is a legitimate field data type value;
                  False if not


return attr in cls._ALL<EOL>

Check a candidate value whether it's one of the valid attributes

        :param attr: (string) candidate value
        :returns: True if the candidate value is a legitimate "special" field
                  attribute; False if not


s = s.strip()<EOL>for pattern in DATETIME_FORMATS:<EOL><INDENT>try:<EOL><INDENT>return datetime.datetime.strptime(s, pattern)<EOL><DEDENT>except ValueError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % (s, '<STR_LIT:U+002CU+0020>'.join(DATETIME_FORMATS)))<EOL>

Parses a textual datetime format and return a Python datetime object.

The supported format is: ``yyyy-mm-dd h:m:s.ms``

The time component is optional.

- hours are 00..23 (no AM/PM)
- minutes are 00..59
- seconds are 00..59
- micro-seconds are 000000..999999

:param s: (string) input time text
:return: (datetime.datetime)


return t.strftime(DATETIME_FORMATS[<NUM_LIT:0>])<EOL>

Turns a datetime object into a string.

:param t: (datetime.datetime)
:return: (string) in default format (see
         :const:`~nupic.data.utils.DATETIME_FORMATS` [0])


return t.strftime(DATETIME_FORMATS[<NUM_LIT:2>])<EOL>

Turns a datetime object into a string ignoring milliseconds.

:param t: (datetime.datetime)
:return: (string) in default format (see
         :const:`~nupic.data.utils.DATETIME_FORMATS` [2])


l = s.lower()<EOL>if l in ("<STR_LIT:true>", "<STR_LIT:t>", "<STR_LIT:1>"):<EOL><INDENT>return True<EOL><DEDENT>if l in ("<STR_LIT:false>", "<STR_LIT:f>", "<STR_LIT:0>"):<EOL><INDENT>return False<EOL><DEDENT>raise Exception("<STR_LIT>" % s)<EOL>

String to boolean

:param s: (string)
:return: (bool)


if f == '<STR_LIT:None>':<EOL><INDENT>return None<EOL><DEDENT>return float(f)<EOL>

Tries to convert input to a float input or returns ``None``.

:param f: (object) thing to convert to a float
:return: (float or ``None``)


if i.strip() == '<STR_LIT:None>' or i.strip() == '<STR_LIT>':<EOL><INDENT>return None<EOL><DEDENT>return int(i)<EOL>

Tries to convert input to a int input or returns ``None``.

:param f: (object) thing to convert to a int
:return: (int or ``None``)


if s is None:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>assert isinstance(s, basestring),"<STR_LIT>" % (basestring, type(s), s)<EOL>s = s.replace('<STR_LIT:\\>', '<STR_LIT>')<EOL>s = s.replace('<STR_LIT:\n>', '<STR_LIT>')<EOL>s = s.replace('<STR_LIT:\t>', '<STR_LIT>')<EOL>s = s.replace('<STR_LIT:U+002C>', '<STR_LIT:\t>')<EOL>return s<EOL>

Escape commas, tabs, newlines and dashes in a string

Commas are encoded as tabs.

:param s: (string) to escape
:returns: (string) escaped string


assert isinstance(s, basestring)<EOL>s = s.replace('<STR_LIT:\t>', '<STR_LIT:U+002C>')<EOL>s = s.replace('<STR_LIT>', '<STR_LIT:U+002C>')<EOL>s = s.replace('<STR_LIT>', '<STR_LIT:\n>')<EOL>s = s.replace('<STR_LIT>', '<STR_LIT:\\>')<EOL>return s<EOL>

Unescapes a string that may contain commas, tabs, newlines and dashes

Commas are decoded from tabs.

:param s: (string) to unescape
:returns: (string) unescaped string


assert isinstance(s, basestring)<EOL>sdr = [int(c) for c in s if c in ("<STR_LIT:0>", "<STR_LIT:1>")]<EOL>if len(sdr) != len(s):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return sdr<EOL>

Parses a string containing only 0's and 1's and return a Python list object.

:param s: (string) string to parse
:returns: (list) SDR out


return "<STR_LIT>".join(str(bit) for bit in sdr)<EOL>

Serialize Python list object containing only 0's and 1's to string.

:param sdr: (list) binary
:returns: (string) SDR out


assert isinstance(s, basestring)<EOL>return [int(i) for i in s.split()]<EOL>

Parse a string of space-separated numbers, returning a Python list.

:param s: (string) to parse
:returns: (list) binary SDR


return "<STR_LIT:U+0020>".join(str(i) for i in listObj)<EOL>

Convert a list of numbers to a string of space-separated values.

:param listObj: (list) to convert
:returns: (string) of space-separated values


<EOL>self._n = n<EOL>self._w = w<EOL>self._num = num<EOL>self._random = Random(seed)<EOL>self._patterns = dict()<EOL>self._generate()<EOL>

@param n   (int)      Number of available bits in pattern
@param w   (int/list) Number of on bits in pattern
                      If list, each pattern will have a `w` randomly
                      selected from the list.
@param num (int)      Number of available patterns


if not number in self._patterns:<EOL><INDENT>raise IndexError("<STR_LIT>")<EOL><DEDENT>return self._patterns[number]<EOL>

Return a pattern for a number.

@param number (int) Number of pattern

@return (set) Indices of on bits


newBits = set()<EOL>for bit in bits:<EOL><INDENT>if self._random.getReal64() < amount:<EOL><INDENT>newBits.add(self._random.getUInt32(self._n))<EOL><DEDENT>else:<EOL><INDENT>newBits.add(bit)<EOL><DEDENT><DEDENT>return newBits<EOL>

Add noise to pattern.

@param bits   (set)   Indices of on bits
@param amount (float) Probability of switching an on bit with a random bit

@return (set) Indices of on bits in noisy pattern


if bit >= self._n:<EOL><INDENT>raise IndexError("<STR_LIT>")<EOL><DEDENT>numbers = set()<EOL>for index, pattern in self._patterns.items():<EOL><INDENT>if bit in pattern:<EOL><INDENT>numbers.add(index)<EOL><DEDENT><DEDENT>return numbers<EOL>

Return the set of pattern numbers that match a bit.

@param bit (int) Index of bit

@return (set) Indices of numbers


numberMap = dict()<EOL>for bit in bits:<EOL><INDENT>numbers = self.numbersForBit(bit)<EOL>for number in numbers:<EOL><INDENT>if not number in numberMap:<EOL><INDENT>numberMap[number] = set()<EOL><DEDENT>numberMap[number].add(bit)<EOL><DEDENT><DEDENT>return numberMap<EOL>

Return a map from number to matching on bits,
for all numbers that match a set of bits.

@param bits (set) Indices of bits

@return (dict) Mapping from number => on bits.


def prettyPrintPattern(self, bits, verbosity=<NUM_LIT:1>):

numberMap = self.numberMapForBits(bits)<EOL>text = "<STR_LIT>"<EOL>numberList = []<EOL>numberItems = sorted(iter(numberMap.items()),<EOL>key=lambda number_bits: len(number_bits[<NUM_LIT:1>]),<EOL>reverse=True)<EOL>for number, bits in numberItems:<EOL><INDENT>if verbosity > <NUM_LIT:2>:<EOL><INDENT>strBits = [str(n) for n in bits]<EOL>numberText = "<STR_LIT>".format(number, "<STR_LIT:U+002C>".join(strBits))<EOL><DEDENT>elif verbosity > <NUM_LIT:1>:<EOL><INDENT>numberText = "<STR_LIT>".format(number, len(bits))<EOL><DEDENT>else:<EOL><INDENT>numberText = str(number)<EOL><DEDENT>numberList.append(numberText)<EOL><DEDENT>text += "<STR_LIT>".format("<STR_LIT:U+002CU+0020>".join(numberList))<EOL>return text<EOL>

Pretty print a pattern.

@param bits      (set) Indices of on bits
@param verbosity (int) Verbosity level

@return (string) Pretty-printed text


candidates = np.array(list(range(self._n)), np.uint32)<EOL>for i in range(self._num):<EOL><INDENT>self._random.shuffle(candidates)<EOL>pattern = candidates[<NUM_LIT:0>:self._getW()]<EOL>self._patterns[i] = set(pattern)<EOL><DEDENT>


w = self._w<EOL>if type(w) is list:<EOL><INDENT>return w[self._random.getUInt32(len(w))]<EOL><DEDENT>else:<EOL><INDENT>return w<EOL><DEDENT>

Gets a value of `w` for use in generating a pattern.


n = self._n<EOL>w = self._w<EOL>assert type(w) is int, "<STR_LIT>"<EOL>for i in range(n / w):<EOL><INDENT>pattern = set(range(i * w, (i+<NUM_LIT:1>) * w))<EOL>self._patterns[i] = pattern<EOL><DEDENT>


<EOL>self.patternMachine = patternMachine<EOL>self._random = Random(seed)<EOL>

@param patternMachine (PatternMachine) Pattern machine instance


sequence = []<EOL>for number in numbers:<EOL><INDENT>if number == None:<EOL><INDENT>sequence.append(number)<EOL><DEDENT>else:<EOL><INDENT>pattern = self.patternMachine.get(number)<EOL>sequence.append(pattern)<EOL><DEDENT><DEDENT>return sequence<EOL>

Generate a sequence from a list of numbers.

Note: Any `None` in the list of numbers is considered a reset.

@param numbers (list) List of numbers

@return (list) Generated sequence


def addSpatialNoise(self, sequence, amount):

newSequence = []<EOL>for pattern in sequence:<EOL><INDENT>if pattern is not None:<EOL><INDENT>pattern = self.patternMachine.addNoise(pattern, amount)<EOL><DEDENT>newSequence.append(pattern)<EOL><DEDENT>return newSequence<EOL>

Add spatial noise to each pattern in the sequence.

@param sequence (list)  Sequence
@param amount   (float) Amount of spatial noise

@return (list) Sequence with spatial noise


def prettyPrintSequence(self, sequence, verbosity=<NUM_LIT:1>):

text = "<STR_LIT>"<EOL>for i in xrange(len(sequence)):<EOL><INDENT>pattern = sequence[i]<EOL>if pattern == None:<EOL><INDENT>text += "<STR_LIT>"<EOL>if i < len(sequence) - <NUM_LIT:1>:<EOL><INDENT>text += "<STR_LIT:\n>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>text += self.patternMachine.prettyPrintPattern(pattern,<EOL>verbosity=verbosity)<EOL><DEDENT><DEDENT>return text<EOL>

Pretty print a sequence.

@param sequence  (list) Sequence
@param verbosity (int)  Verbosity level

@return (string) Pretty-printed text


def generateNumbers(self, numSequences, sequenceLength, sharedRange=None):

numbers = []<EOL>if sharedRange:<EOL><INDENT>sharedStart, sharedEnd = sharedRange<EOL>sharedLength = sharedEnd - sharedStart<EOL>sharedNumbers = range(numSequences * sequenceLength,<EOL>numSequences * sequenceLength + sharedLength)<EOL><DEDENT>for i in xrange(numSequences):<EOL><INDENT>start = i * sequenceLength<EOL>newNumbers = np.array(range(start, start + sequenceLength), np.uint32)<EOL>self._random.shuffle(newNumbers)<EOL>newNumbers = list(newNumbers)<EOL>if sharedRange is not None:<EOL><INDENT>newNumbers[sharedStart:sharedEnd] = sharedNumbers<EOL><DEDENT>numbers += newNumbers<EOL>numbers.append(None)<EOL><DEDENT>return numbers<EOL>

@param numSequences   (int)   Number of sequences to return,
                              separated by None
@param sequenceLength (int)   Length of each sequence
@param sharedRange    (tuple) (start index, end index) indicating range of
                              shared subsequence in each sequence
                              (None if no shared subsequences)
@return (list) Numbers representing sequences


description = {'<STR_LIT:name>':self.name, '<STR_LIT>':[f.name for f in self.fields],'<STR_LIT>':[f.numRecords for f in self.fields]}<EOL>return description<EOL>


rand.seed(seed)<EOL>np.random.seed(seed)<EOL>


def addField(self, name, fieldParams, encoderParams):

assert fieldParams is not None and'<STR_LIT:type>' in fieldParams<EOL>dataClassName = fieldParams.pop('<STR_LIT:type>')<EOL>try:<EOL><INDENT>dataClass=eval(dataClassName)(fieldParams)<EOL><DEDENT>except TypeError as e:<EOL><INDENT>print(("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (dataClass, fieldParams)))<EOL>raise<EOL><DEDENT>encoderParams['<STR_LIT>']=dataClass<EOL>encoderParams['<STR_LIT>']=dataClassName<EOL>fieldIndex = self.defineField(name, encoderParams)<EOL>

Add a single field to the dataset.
        Parameters:
        -------------------------------------------------------------------
        name:             The user-specified name of the field
        fieldSpec:        A list of one or more dictionaries specifying parameters
                          to be used for dataClass initialization. Each dict must
                          contain the key 'type' that specifies a distribution for
                          the values in this field
        encoderParams:    Parameters for the field encoder


assert all(x in field for x in ['<STR_LIT:name>', '<STR_LIT>', '<STR_LIT>'] for fieldin fieldsInfo)<EOL>for spec in fieldsInfo:<EOL><INDENT>self.addField(spec.pop('<STR_LIT:name>'), spec.pop('<STR_LIT>'), spec.pop('<STR_LIT>'))<EOL><DEDENT>

Add multiple fields to the dataset.
        Parameters:
        -------------------------------------------------------------------
        fieldsInfo:       A list of dictionaries, containing a field name, specs for
                          the data classes and encoder params for the corresponding
                          field.


def defineField(self, name, encoderParams=None):

self.fields.append(_field(name, encoderParams))<EOL>return len(self.fields)-<NUM_LIT:1><EOL>

Initialize field using relevant encoder parameters.
        Parameters:
        -------------------------------------------------------------------
        name:                 Field name
        encoderParams:        Parameters for the encoder.

        Returns the index of the field


assert len(self.fields)>index<EOL>self.fields[index].flag=flag<EOL>

Set flag for field at index. Flags are special characters such as 'S' for
        sequence or 'T' for timestamp.
        Parameters:
        --------------------------------------------------------------------
        index:            index of field whose flag is being set
        flag:             special character


assert(len(record)==len(self.fields))<EOL>if record is not None:<EOL><INDENT>for x in range(len(self.fields)):<EOL><INDENT>self.fields[x].addValue(record[x])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for field in self.fields:<EOL><INDENT>field.addValue(field.dataClass.getNext())<EOL><DEDENT><DEDENT>

Generate a record. Each value is stored in its respective field.
        Parameters:
        --------------------------------------------------------------------
        record:       A 1-D array containing as many values as the number of fields
        fields:       An object of the class field that specifies the characteristics
                      of each value in the record
        Assertion:
        --------------------------------------------------------------------
        len(record)==len(fields):   A value for each field must be specified.
                                    Replace missing values of any type by
                                    SENTINEL_VALUE_FOR_MISSING_DATA

        This method supports external classes but not combination of classes.


if self.verbosity><NUM_LIT:0>: print('<STR_LIT>', len(records), '<STR_LIT>')<EOL>for record in records:<EOL><INDENT>self.generateRecord(record)<EOL><DEDENT>

Generate multiple records. Refer to definition for generateRecord


if n is None:<EOL><INDENT>assert len(self.fields)><NUM_LIT:0><EOL>n = self.fields[<NUM_LIT:0>].numRecords-<NUM_LIT:1><EOL><DEDENT>assert (all(field.numRecords>n for field in self.fields))<EOL>record = [field.values[n] for field in self.fields]<EOL>return record<EOL>


values=[]<EOL>numRecords = self.fields[<NUM_LIT:0>].numRecords<EOL>assert (all(field.numRecords==numRecords for field in self.fields))<EOL>for x in range(numRecords):<EOL><INDENT>values.append(self.getRecord(x))<EOL><DEDENT>return values<EOL>


def encodeRecord(self, record, toBeAdded=True):

encoding=[self.fields[i].encodeValue(record[i], toBeAdded) for i inrange(len(self.fields))]<EOL>return encoding<EOL>

Encode a record as a sparse distributed representation
        Parameters:
        --------------------------------------------------------------------
        record:        Record to be encoded
        toBeAdded:     Whether the encodings corresponding to the record are added to
                       the corresponding fields


def encodeAllRecords(self, records=None, toBeAdded=True):

if records is None:<EOL><INDENT>records = self.getAllRecords()<EOL><DEDENT>if self.verbosity><NUM_LIT:0>: print('<STR_LIT>', len(records), '<STR_LIT>')<EOL>encodings = [self.encodeRecord(record, toBeAdded) for record in records]<EOL>return encodings<EOL>

Encodes a list of records.
        Parameters:
        --------------------------------------------------------------------
        records:      One or more records. (i,j)th element of this 2D array
                      specifies the value at field j of record i.
                      If unspecified, records previously generated and stored are
                      used.
        toBeAdded:    Whether the encodings corresponding to the record are added to
                      the corresponding fields


def addValueToField(self, i, value=None):

assert(len(self.fields)>i)<EOL>if value is None:<EOL><INDENT>value = self.fields[i].dataClass.getNext()<EOL>self.fields[i].addValue(value)<EOL>return value<EOL><DEDENT>else: self.fields[i].addValue(value)<EOL>


def addValuesToField(self, i, numValues):

assert(len(self.fields)>i)<EOL>values = [self.addValueToField(i) for n in range(numValues)]<EOL>return values<EOL>


assert len(self.fields)>i<EOL>assert self.fields[i].numRecords>j<EOL>encoding = self.fields[i].encodings[j]<EOL>return encoding<EOL>

Returns the sdr for jth value at column i


assert all(field.numRecords>n for field in self.fields)<EOL>encoding = np.concatenate([field.encoder.encode(SENTINEL_VALUE_FOR_MISSING_DATA)if field.isPredictedField else field.encodings[n] for field in self.fields])<EOL>return encoding<EOL>

Returns the nth encoding with the predictedField zeroed out


n = sum([field.n for field in self.fields])<EOL>return n<EOL>

Returns the cumulative n for all the fields in the dataset


w = sum([field.w for field in self.fields])<EOL>return w<EOL>

Returns the cumulative w for all the fields in the dataset


assert (all(field.numEncodings>n for field in self.fields))<EOL>encoding = np.concatenate([field.encodings[n] for field in self.fields])<EOL>return encoding<EOL>


numEncodings=self.fields[<NUM_LIT:0>].numEncodings<EOL>assert (all(field.numEncodings==numEncodings for field in self.fields))<EOL>encodings = [self.getEncoding(index) for index in range(numEncodings)]<EOL>return encodings<EOL>


names = [field.name for field in self.fields]<EOL>return names<EOL>


flags = [field.flag for field in self.fields]<EOL>return flags<EOL>


dataTypes = [field.dataType for field in self.fields]<EOL>return dataTypes<EOL>


descriptions = [field.getDescription() for field in self.fields]<EOL>return descriptions<EOL>


numRecords = self.fields[<NUM_LIT:0>].numRecords<EOL>assert (all(field.numRecords==numRecords for field in self.fields))<EOL>import csv<EOL>with open(path+'<STR_LIT>', '<STR_LIT:wb>') as f:<EOL><INDENT>writer = csv.writer(f)<EOL>writer.writerow(self.getAllFieldNames())<EOL>writer.writerow(self.getAllDataTypes())<EOL>writer.writerow(self.getAllFlags())<EOL>writer.writerows(self.getAllRecords())<EOL><DEDENT>if self.verbosity><NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>', numRecords,'<STR_LIT>',path,'<STR_LIT>')<EOL><DEDENT>

Export all the records into a csv file in numenta format.

        Example header format:
        fieldName1    fieldName2    fieldName3
        date          string        float
        T             S

        Parameters:
        --------------------------------------------------------------------
        path:      Relative path of the file to which the records are to be exported


for field in self.fields:<EOL><INDENT>field.encodings, field.values=[], []<EOL>field.numRecords, field.numEncodings= (<NUM_LIT:0>, <NUM_LIT:0>)<EOL><DEDENT>


self.name=name<EOL>self.n, self.w = (<NUM_LIT:100>, <NUM_LIT:15>)<EOL>self.encoderType,self.dataType,self.dataClassName  = (None, None, None)<EOL>self.flag='<STR_LIT>'<EOL>self.isPredictedField=False<EOL>if encoderSpec is not None:<EOL><INDENT>if '<STR_LIT:n>' in encoderSpec: self.n = encoderSpec.pop('<STR_LIT:n>')<EOL>if '<STR_LIT:w>' in encoderSpec: self.w = encoderSpec.pop('<STR_LIT:w>')<EOL>if '<STR_LIT>' in encoderSpec: self.flag = encoderSpec.pop('<STR_LIT>')<EOL>if '<STR_LIT>' in encoderSpec: self.isPredictedField= encoderSpec.pop('<STR_LIT>')<EOL>if '<STR_LIT>' in encoderSpec: self.dataClass= encoderSpec.pop('<STR_LIT>')<EOL>if '<STR_LIT>' in encoderSpec: self.dataClassName= encoderSpec.pop('<STR_LIT>')<EOL>if '<STR_LIT>' in encoderSpec: self.dataType = encoderSpec.pop('<STR_LIT>')<EOL>if '<STR_LIT>' in encoderSpec: self.encoderType= encoderSpec.pop('<STR_LIT>')<EOL><DEDENT>if self.dataType is None and self.encoderType is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>assert(self.dataType is not None or self.encoderType is not None)<EOL>if self.dataType is None or self.encoderType is None:<EOL><INDENT>self._setTypes(encoderSpec)<EOL><DEDENT>self._initializeEncoders(encoderSpec)<EOL>self.encodings=[]<EOL>self.values=[]<EOL>self.numRecords=<NUM_LIT:0><EOL>self.numEncodings=<NUM_LIT:0><EOL>

Initialize a field with various parameters such as n, w, flag, dataType,
        encoderType, and tag predicted field.


for v in values:<EOL><INDENT>self.addValue(v)<EOL><DEDENT>


self.values.append(value)<EOL>self.numRecords+=<NUM_LIT:1><EOL>


def encodeValue(self, value, toBeAdded=True):

encodedValue = np.array(self.encoder.encode(value), dtype=realDType)<EOL>if toBeAdded:<EOL><INDENT>self.encodings.append(encodedValue)<EOL>self.numEncodings+=<NUM_LIT:1><EOL><DEDENT>return encodedValue<EOL>

Value is encoded as a sdr using the encoding parameters of the Field


if self.encoderType is None:<EOL><INDENT>if self.dataType in ['<STR_LIT:int>','<STR_LIT:float>']:<EOL><INDENT>self.encoderType='<STR_LIT>'<EOL><DEDENT>elif self.dataType=='<STR_LIT:string>':<EOL><INDENT>self.encoderType='<STR_LIT>'<EOL><DEDENT>elif self.dataType in ['<STR_LIT:date>', '<STR_LIT>']:<EOL><INDENT>self.encoderType='<STR_LIT:date>'<EOL><DEDENT><DEDENT>if self.dataType is None:<EOL><INDENT>if self.encoderType in ['<STR_LIT>','<STR_LIT>']:<EOL><INDENT>self.dataType='<STR_LIT:float>'<EOL><DEDENT>elif self.encoderType in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>self.dataType='<STR_LIT:string>'<EOL><DEDENT>elif self.encoderType in ['<STR_LIT:date>', '<STR_LIT>']:<EOL><INDENT>self.dataType='<STR_LIT>'<EOL><DEDENT><DEDENT>

Set up the dataTypes and initialize encoders