if isinstance(obj, (set, tuple, list)):<EOL><INDENT>return tuple([make_hash(e) for e in obj])<EOL><DEDENT>elif not isinstance(obj, dict):<EOL><INDENT>return hash(obj)<EOL><DEDENT>new_obj = copy.deepcopy(obj)<EOL>for k, v in new_obj.items():<EOL><INDENT>new_obj[k] = make_hash(v)<EOL><DEDENT>return hash(tuple(frozenset(new_obj.items())))<EOL>

Makes a hash from a dictionary, list, tuple or set to any level,
that contains only other hashable types (including any lists,
tuples, sets, and dictionaries).  See second answer (not the
accepted answer):
http://stackoverflow.com/questions/5884066/hashing-a-python-dictionary


make_absolute_paths(config)<EOL>pipeline_config = config['<STR_LIT>']<EOL>pipeline_config['<STR_LIT>'] = make_hash(config)<EOL>pipeline_config['<STR_LIT>'] = json.dumps(config)<EOL>logger.debug('<STR_LIT>'<EOL>.format(pipeline_config['<STR_LIT>'], config))<EOL>die = False<EOL>for pathstr in pipeline_config.get('<STR_LIT>', {}).itervalues():<EOL><INDENT>if pathstr not in sys.path:<EOL><INDENT>sys.path.append(pathstr)<EOL><DEDENT><DEDENT>for modname in pipeline_config.get('<STR_LIT>', {}).itervalues():<EOL><INDENT>try:<EOL><INDENT>m = importlib.import_module(modname)<EOL>if not m:<EOL><INDENT>logger.critical('<STR_LIT>', modname)<EOL>die = True<EOL>continue<EOL><DEDENT>if hasattr(m, '<STR_LIT>'):<EOL><INDENT>m.setup()<EOL>logger.debug('<STR_LIT>', modname)<EOL><DEDENT>else:<EOL><INDENT>logger.debug('<STR_LIT>', modname)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>logger.critical('<STR_LIT>', modname, exc_info=True)<EOL>die = True<EOL><DEDENT><DEDENT>if die:<EOL><INDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

setup the config and load external modules

    This updates 'config' as follows:

    * All paths are replaced with absolute paths
    * A hash and JSON dump of the config are stored in the config
    * If 'pythonpath' is in the config, it is added to sys.path
    * If 'setup_modules' is in the config, all modules named in it are loaded


<EOL>creation_time = '<STR_LIT>'<EOL>correct_time = <NUM_LIT><EOL>if not os.path.isabs(path_to_original):<EOL><INDENT>path_to_original = os.path.join(os.getcwd(), path_to_original)<EOL><DEDENT>for label_id in range(<NUM_LIT>):<EOL><INDENT>dir_path = os.path.join(path_to_original, str(label_id))<EOL>fnames = os.listdir(dir_path)<EOL>fnames.sort()<EOL>for fname in fnames:<EOL><INDENT>stream_item = streamcorpus.make_stream_item(<EOL>creation_time, <EOL>os.path.join(<EOL>'<STR_LIT>', str(label_id), fname))<EOL>if int(stream_item.stream_time.epoch_ticks) != correct_time:<EOL><INDENT>raise PipelineBaseException('<STR_LIT>'% (creation_time, stream_item.stream_time.epoch_ticks,<EOL>correct_time))<EOL><DEDENT>stream_item.source = '<STR_LIT>'<EOL>body = streamcorpus.ContentItem()<EOL>raw_string = open(os.path.join(dir_path, fname)).read()<EOL>body.clean_visible = str(raw_string).encode('<STR_LIT:utf8>')<EOL>stream_item.body = body<EOL>stream_item.body.language = streamcorpus.Language(code='<STR_LIT>', name='<STR_LIT>')<EOL>anno = streamcorpus.Annotator()<EOL>anno.annotator_id = '<STR_LIT>'<EOL>anno.annotation_time = stream_item.stream_time<EOL>rating = streamcorpus.Rating()<EOL>rating.annotator = anno<EOL>rating.target = streamcorpus.Target(target_id = str(label_id)) <EOL>rating.contains_mention = True<EOL>rating.mentions = ['<STR_LIT>', '<STR_LIT>']<EOL>streamcorpus.add_annotation(stream_item, rating)<EOL>yield stream_item<EOL><DEDENT><DEDENT>

This _looks_ like a Chunk only in that it generates StreamItem
instances when iterated upon.


return generate_john_smith_chunk(i_str)<EOL>

Returns a kba.pipeline "reader" that generates a single
streamcorpus.Chunk file containing the John Smith corpus.

:returns function:


path_to_input_file = os.path.abspath(path_to_input_file)<EOL>with open(path_to_input_file) as f:<EOL><INDENT>metadata = yaml.load(f)<EOL>entities = metadata.pop('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' not in metadata:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>root_path = metadata['<STR_LIT>']<EOL>if root_path is None or len(root_path.strip()) == <NUM_LIT:0>:<EOL><INDENT>root_path = os.getcwd()<EOL><DEDENT>if '<STR_LIT:source>' not in metadata:<EOL><INDENT>metadata['<STR_LIT:source>'] = '<STR_LIT>'<EOL>logger.warn('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' not in metadata:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>docs = defaultdict(list)<EOL>abs_urls = dict()<EOL>for entity in entities:<EOL><INDENT>paths = entity['<STR_LIT>']<EOL>if not isinstance(paths, list):<EOL><INDENT>paths = [paths]<EOL><DEDENT>paths.extend(entity.get('<STR_LIT>', []))<EOL>external_profiles = set()<EOL>for ext_profile in entity.get('<STR_LIT>', []):<EOL><INDENT>external_profiles.add(ext_profile['<STR_LIT>'])<EOL><DEDENT>visited_paths = []<EOL>for path in paths:<EOL><INDENT>abs_url = None<EOL>if isinstance(path, dict):<EOL><INDENT>abs_url = path['<STR_LIT>']<EOL>path = path['<STR_LIT>']<EOL><DEDENT>path = str(path)<EOL>if not os.path.isabs(path):<EOL><INDENT>path = os.path.join(root_path, path)<EOL><DEDENT>if os.path.isdir(path):<EOL><INDENT>for dirpath, dirnames, filenames in os.walk(path):<EOL><INDENT>for fname in filenames:<EOL><INDENT>fpath = os.path.join(dirpath, fname)<EOL>if fpath not in visited_paths:<EOL><INDENT>is_profile = bool(fpath in external_profiles)<EOL>sub_path = fpath[len(path):]<EOL>if abs_url is None:<EOL><INDENT>_abs_url = '<STR_LIT>' + sub_path<EOL><DEDENT>else:<EOL><INDENT>_abs_url = abs_url + '<STR_LIT:#>' + urllib.parse.quote(sub_path)<EOL><DEDENT>docs[fpath].append((entity, is_profile))<EOL>abs_urls[fpath] = _abs_url<EOL>visited_paths.append(fpath)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if path not in visited_paths:<EOL><INDENT>is_profile = bool(path in external_profiles)<EOL>if abs_url is None:<EOL><INDENT>raise Exception('<STR_LIT>' % path)<EOL><DEDENT>docs[path].append((entity, is_profile))<EOL>abs_urls[path] = abs_url<EOL>visited_paths.append(path)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for path in docs:<EOL><INDENT>yield self._make_stream_item(path, metadata, abs_urls[path], docs[path])<EOL><DEDENT>


previous_end = <NUM_LIT:0><EOL>clean_visible = clean_visible.decode('<STR_LIT:utf8>')<EOL>for start, end in self.sentence_tokenizer.span_tokenize(clean_visible):<EOL><INDENT>if start < previous_end:<EOL><INDENT>start = previous_end<EOL>if start > end:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>try:<EOL><INDENT>label = self.label_index.find_le(end)<EOL><DEDENT>except ValueError:<EOL><INDENT>label = None<EOL><DEDENT>if label:<EOL><INDENT>off = label.offsets[OffsetType.CHARS]<EOL>end = max(off.first + off.length, end)<EOL><DEDENT>previous_end = end<EOL>sent_str = clean_visible[start:end]<EOL>yield start, end, sent_str<EOL><DEDENT>

generate strings identified as sentences


labels = stream_item.body.labels.get(self.annotator_id)<EOL>if not labels:<EOL><INDENT>labels = []<EOL><DEDENT>self.label_index = SortedCollection(<EOL>[l for l in labels if OffsetType.CHARS in l.offsets],<EOL>key=lambda label: label.offsets[OffsetType.CHARS].first)<EOL>


self.make_label_index(stream_item)<EOL>sentences = []<EOL>token_num = <NUM_LIT:0><EOL>new_mention_id = <NUM_LIT:0><EOL>for sent_start, sent_end, sent_str in self._sentences(<EOL>stream_item.body.clean_visible):<EOL><INDENT>assert isinstance(sent_str, unicode)<EOL>sent = Sentence()<EOL>sentence_pos = <NUM_LIT:0><EOL>for start, end in self.word_tokenizer.span_tokenize(sent_str):<EOL><INDENT>token_str = sent_str[start:end].encode('<STR_LIT:utf8>')<EOL>tok = Token(<EOL>token_num=token_num,<EOL>token=token_str,<EOL>sentence_pos=sentence_pos,<EOL>)<EOL>tok.offsets[OffsetType.CHARS] = Offset(<EOL>type=OffsetType.CHARS,<EOL>first=sent_start + start,<EOL>length=end - start,<EOL>)<EOL>try:<EOL><INDENT>label = self.label_index.find_le(sent_start + start)<EOL><DEDENT>except ValueError:<EOL><INDENT>label = None<EOL><DEDENT>if label:<EOL><INDENT>off = label.offsets[OffsetType.CHARS]<EOL>if off.first + off.length > sent_start + start:<EOL><INDENT>streamcorpus.add_annotation(tok, label)<EOL>logger.debug('<STR_LIT>',<EOL>tok.token, label.target.target_id)<EOL>if label in self.label_to_mention_id:<EOL><INDENT>mention_id = self.label_to_mention_id[label]<EOL><DEDENT>else:<EOL><INDENT>mention_id = new_mention_id<EOL>new_mention_id += <NUM_LIT:1><EOL>self.label_to_mention_id[label] = mention_id<EOL><DEDENT>tok.mention_id = mention_id<EOL><DEDENT><DEDENT>token_num += <NUM_LIT:1><EOL>sentence_pos += <NUM_LIT:1><EOL>sent.tokens.append(tok)<EOL><DEDENT>sentences.append(sent)<EOL><DEDENT>return sentences<EOL>


super(PipelineFactory, self).__init__()<EOL>self.registry = registry<EOL>self.lock = threading.Lock()<EOL>self.tmp_dir_suffix = None<EOL>

Create a pipeline factory.

        :param dict config: top-level "streamcorpus_pipeline" configuration
        :param registry: registry of stages
        :type registry: :class:`~streamcorpus_pipeline.stages.StageRegistry`


def create(self, stage, scp_config, config=None):

<EOL>if isinstance(stage, str):<EOL><INDENT>stage_name = stage<EOL>stage_obj = self.registry[stage_name]<EOL><DEDENT>else:<EOL><INDENT>stage_name = getattr(stage, '<STR_LIT>', stage.__name__)<EOL>stage_obj = stage<EOL><DEDENT>if config is None:<EOL><INDENT>config = scp_config.get(stage_name, None)<EOL><DEDENT>if config is None:<EOL><INDENT>config = getattr(stage_obj, '<STR_LIT>', {})<EOL><DEDENT>config = dict(config)<EOL>if self.tmp_dir_suffix is None:<EOL><INDENT>config['<STR_LIT>'] = scp_config['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>config['<STR_LIT>'] = os.path.join(scp_config['<STR_LIT>'],<EOL>self.tmp_dir_suffix)<EOL><DEDENT>config['<STR_LIT>'] = scp_config['<STR_LIT>']<EOL>return stage_obj(config)<EOL>

Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`


return self.create(config[name], config)<EOL>

Create a single indirect stage.

        `name` should be the name of a config item that holds the
        name of a stage, for instance, ``reader``.  This looks up
        the name of that stage, then creates and returns the
        stage named.  For instance, if the config says

        .. code-block:: yaml

            reader: from_local_chunks

        then calling ``self._init_stage(scp_config, 'reader')`` will
        return a new instance of the
        :class:`~streamcorpus_pipeline._local_storage.from_local_chunks`
        stage.

        :param dict config: `streamcorpus_pipeline` configuration block
        :param str name: name of stage name entry
        :return: new instance of the stage


if name not in config:<EOL><INDENT>return []<EOL><DEDENT>return [self.create(stage, config) for stage in config[name]]<EOL>

Create a list of indirect stages.

        `name` should be the name of a config item that holds a list
        of names of stages, for instance, ``writers``.  This looks up
        the names of those stages, then creates and returns the
        corresponding list of stage objects.  For instance, if the
        config says

        .. code-block:: yaml

            incremental_transforms: [clean_html, clean_visible]

        then calling ``self._init_stages(scp_config,
        'incremental_transforms')`` will return a list of the two
        named stage instances.

        :param dict config: `streamcorpus_pipeline` configuration block
        :param str name: name of the stage name list entry
        :return: list of new stage instances


reader = self._init_stage(config, '<STR_LIT>')<EOL>incremental_transforms = self._init_stages(<EOL>config, '<STR_LIT>')<EOL>batch_transforms = self._init_stages(config, '<STR_LIT>')<EOL>post_batch_incremental_transforms = self._init_stages(<EOL>config, '<STR_LIT>')<EOL>writers = self._init_stages(config, '<STR_LIT>')<EOL>tmp_dir_path = os.path.join(config['<STR_LIT>'],<EOL>self.tmp_dir_suffix)<EOL>return (reader, incremental_transforms, batch_transforms,<EOL>post_batch_incremental_transforms, writers, tmp_dir_path)<EOL>

Create stages that are used for the pipeline.

        :param dict config: `streamcorpus_pipeline` configuration
        :return: tuple of (reader, incremental transforms, batch
          transforms, post-batch incremental transforms, writers,
          temporary directory)


tmp_dir_suffix = self.tmp_dir_suffix<EOL>if tmp_dir_suffix is None:<EOL><INDENT>with self.lock:<EOL><INDENT>self.tmp_dir_suffix = str(uuid.uuid4())<EOL>try:<EOL><INDENT>(reader, incremental_transforms, batch_transforms,<EOL>pbi_transforms, writers,<EOL>tmp_dir_path) = self._init_all_stages(config)<EOL><DEDENT>finally:<EOL><INDENT>self.tmp_dir_suffix = None<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>(reader, incremental_transforms, batch_transforms,<EOL>pbi_transforms, writers,<EOL>tmp_dir_path) = self._init_all_stages(config)<EOL><DEDENT>return Pipeline(<EOL>rate_log_interval=config['<STR_LIT>'],<EOL>input_item_limit=config.get('<STR_LIT>'),<EOL>cleanup_tmp_files=config['<STR_LIT>'],<EOL>tmp_dir_path=tmp_dir_path,<EOL>assert_single_source=config['<STR_LIT>'],<EOL>output_chunk_max_count=config.get('<STR_LIT>'),<EOL>output_max_clean_visible_bytes=config.get(<EOL>'<STR_LIT>'),<EOL>reader=reader,<EOL>incremental_transforms=incremental_transforms,<EOL>batch_transforms=batch_transforms,<EOL>post_batch_incremental_transforms=pbi_transforms,<EOL>writers=writers,<EOL>)<EOL>

Create a :class:`Pipeline`.

        Pass in the configuration under the ``streamcorpus_pipeline``
        block, not the top-level configuration that contains it.

        If :attr:`tmp_dir_suffix` is :const:`None`, then locks the
        factory and creates stages with a temporary (UUID) value.  If
        the configuration has `cleanup_tmp_files` set to :const:`True`
        (the default) then executing the resulting pipeline will clean
        up the directory afterwards.

        :param dict config: `streamcorpus_pipeline` configuration
        :return: new pipeline instance


def __init__(self, rate_log_interval, input_item_limit,<EOL>cleanup_tmp_files, tmp_dir_path, assert_single_source,<EOL>output_chunk_max_count, output_max_clean_visible_bytes,<EOL>reader, incremental_transforms, batch_transforms,<EOL>post_batch_incremental_transforms, writers):

self.rate_log_interval = rate_log_interval<EOL>self.input_item_limit = input_item_limit<EOL>self.cleanup_tmp_files = cleanup_tmp_files<EOL>self.tmp_dir_path = tmp_dir_path<EOL>self.assert_single_source = assert_single_source<EOL>self.output_chunk_max_count = output_chunk_max_count<EOL>self.output_max_clean_visible_bytes = output_max_clean_visible_bytes<EOL>self.reader = reader<EOL>self.incremental_transforms = incremental_transforms<EOL>self.batch_transforms = batch_transforms<EOL>self.pbi_stages = post_batch_incremental_transforms<EOL>self.writers = writers<EOL>self.t_chunk = None<EOL>self.context = dict(<EOL>i_str=None,<EOL>data=None,<EOL>)<EOL>self.work_unit = None<EOL>

Create a new pipeline object.

        .. todo:: make this callable with just the lists of stages
                  and give sane (language-level) defaults for the rest

        :param int rate_log_interval: print progress every time this
          many input items have been processed
        :param int input_item_limit: stop after this many items
        :param bool cleanup_tmp_files: delete `tmp_dir_path` after
          execution if true
        :param str tmp_dir_path: path for intermediate files
        :param bool assert_single_source: require all items to have
          the same source value if true
        :param int output_chunk_max_count: restart output after
          writing this many items
        :param int output_max_clean_visible_bytes: restart output after
          writing this much content
        :param callable reader: reader stage object
        :param incremental_transforms: single-item transformation stages
        :paramtype incremental_transforms: list of callable
        :param batch_transforms: chunk-file transformation stages
        :paramtype batch_transforms: list of callable
        :param post_batch_incremental_transforms: single-item transformation
          stages
        :paramtype post_batch_incremental_transforms: list of callable
        :param writers: output stages
        :paramtype writers: list of callable


self.work_unit = work_unit<EOL>i_str = work_unit.key<EOL>start_count = work_unit.data['<STR_LIT>']<EOL>start_chunk_time = work_unit.data['<STR_LIT>']<EOL>self.run(i_str, start_count, start_chunk_time)<EOL>

Process a :class:`coordinate.WorkUnit`.

        The work unit's key is taken as the input file name.  The
        data should have ``start_count`` and ``start_chunk_time``
        values, which are passed on to :meth:`run`.

        :param work_unit: work unit to process
        :paramtype work_unit: :class:`coordinate.WorkUnit`
        :return: number of stream items processed


def run(self, i_str, start_count=<NUM_LIT:0>, start_chunk_time=None):

try:<EOL><INDENT>if not os.path.exists(self.tmp_dir_path):<EOL><INDENT>os.makedirs(self.tmp_dir_path)<EOL><DEDENT>if start_chunk_time is None:<EOL><INDENT>start_chunk_time = time.time()<EOL><DEDENT>i_chunk = self.reader(i_str)<EOL>t_path = None<EOL>len_clean_visible = <NUM_LIT:0><EOL>sources = set()<EOL>next_idx = <NUM_LIT:0><EOL>input_item_count = <NUM_LIT:0><EOL>for si in i_chunk:<EOL><INDENT>next_idx += <NUM_LIT:1><EOL>if gevent:<EOL><INDENT>gevent.sleep(<NUM_LIT:0>)<EOL><DEDENT>if next_idx <= start_count:<EOL><INDENT>continue<EOL><DEDENT>if next_idx % self.rate_log_interval == <NUM_LIT:0>:<EOL><INDENT>elapsed = time.time() - start_chunk_time<EOL>if elapsed > <NUM_LIT:0>:<EOL><INDENT>rate = float(next_idx) / elapsed<EOL>logger.info('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>next_idx - start_count, elapsed, rate,<EOL>i_str)<EOL><DEDENT><DEDENT>if not self.t_chunk:<EOL><INDENT>t_path = os.path.join(self.tmp_dir_path,<EOL>'<STR_LIT>' % uuid.uuid4().hex)<EOL>self.t_chunk = streamcorpus.Chunk(path=t_path, mode='<STR_LIT:wb>')<EOL>assert self.t_chunk.message == streamcorpus.StreamItem_v0_3_0, self.t_chunk.message<EOL><DEDENT>si = self._run_incremental_transforms(<EOL>si, self.incremental_transforms)<EOL>if si:<EOL><INDENT>sources.add(si.source)<EOL>if self.assert_single_source and len(sources) != <NUM_LIT:1>:<EOL><INDENT>raise InvalidStreamItem(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(si.stream_id, si.source, sources))<EOL><DEDENT><DEDENT>if si and si.body and si.body.clean_visible:<EOL><INDENT>len_clean_visible += len(si.body.clean_visible)<EOL><DEDENT>if ((self.output_chunk_max_count is not None and<EOL>len(self.t_chunk) == self.output_chunk_max_count)):<EOL><INDENT>logger.info('<STR_LIT>',<EOL>len(self.t_chunk), next_idx)<EOL>self._process_output_chunk(<EOL>start_count, next_idx, sources, i_str, t_path)<EOL>start_count = next_idx<EOL><DEDENT>elif (self.output_max_clean_visible_bytes is not None and<EOL>len_clean_visible >=<EOL>self.output_chunk_max_clean_visible_bytes):<EOL><INDENT>logger.info(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self.output_chunk_max_clean_visible_bytes,<EOL>len_clean_visible)<EOL>len_clean_visible = <NUM_LIT:0><EOL>self._process_output_chunk(<EOL>start_count, next_idx, sources, i_str, t_path)<EOL>start_count = next_idx<EOL><DEDENT>input_item_count += <NUM_LIT:1><EOL>if (((self.input_item_limit is not None) and<EOL>(input_item_count > self.input_item_limit))):<EOL><INDENT>break<EOL><DEDENT><DEDENT>if self.t_chunk is not None:<EOL><INDENT>self._process_output_chunk(<EOL>start_count, next_idx, sources, i_str, t_path)<EOL><DEDENT>return next_idx<EOL><DEDENT>finally:<EOL><INDENT>if self.t_chunk is not None:<EOL><INDENT>self.t_chunk.close()<EOL><DEDENT>for transform in self.batch_transforms:<EOL><INDENT>transform.shutdown()<EOL><DEDENT>if self.cleanup_tmp_files:<EOL><INDENT>rmtree(self.tmp_dir_path)<EOL><DEDENT><DEDENT>

Run the pipeline.

        This runs all of the steps described in the pipeline constructor,
        reading from some input and writing to some output.

        :param str i_str: name of the input file, or other reader-specific
          description of where to get input
        :param int start_count: index of the first stream item
        :param int start_chunk_time: timestamp for the first stream item


def _process_output_chunk(self, start_count, next_idx, sources, i_str,<EOL>t_path):

if not self.t_chunk:<EOL><INDENT>return []<EOL><DEDENT>self.t_chunk.close()<EOL>o_paths = None<EOL>if len(self.t_chunk) > <NUM_LIT:0>:<EOL><INDENT>logger.info('<STR_LIT>',<EOL>len(self.t_chunk))<EOL>self._run_batch_transforms(t_path)<EOL>self._maybe_run_post_batch_incremental_transforms(t_path)<EOL>if (self.t_chunk) and (len(self.t_chunk) >= <NUM_LIT:0>):<EOL><INDENT>o_paths = self._run_writers(start_count, next_idx, sources,<EOL>i_str, t_path)<EOL><DEDENT><DEDENT>self.t_chunk = None<EOL>if self.work_unit and o_paths:<EOL><INDENT>old_o_paths = self.work_unit.data.get('<STR_LIT>', [])<EOL>o_paths = old_o_paths + o_paths<EOL>self.work_unit.data['<STR_LIT>'] = next_idx<EOL>self.work_unit.data['<STR_LIT>'] = o_paths<EOL>self.work_unit.update()<EOL><DEDENT>

for the current output chunk (which should be open):
  1. run batch transforms
  2. run post-batch incremental transforms
  3. run 'writers' to load-out the data to files or other storage
return list of paths that writers wrote to


for transform in self.batch_transforms:<EOL><INDENT>transform.process_path(chunk_path)<EOL><DEDENT>

Run all of the batch transforms over some intermediate chunk.


def _run_writers(self, start_count, next_idx, sources, i_str, t_path):

<EOL>name_info = dict(<EOL>first=start_count,<EOL>source=sources.pop(),<EOL>)<EOL>all_o_paths = []<EOL>for writer in self.writers:<EOL><INDENT>logger.debug('<STR_LIT>', writer, i_str, name_info)<EOL>o_paths = writer(t_path, name_info, i_str)<EOL>logger.debug('<STR_LIT>',<EOL>start_count, next_idx - <NUM_LIT:1>, i_str, o_paths)<EOL>all_o_paths += o_paths<EOL><DEDENT>return all_o_paths<EOL>

Run all of the writers over some intermediate chunk.

        :param int start_count: index of the first item
        :param int next_idx: index of the next item (after the last
          item in this chunk)
        :param list sources: source strings included in this chunk
          (usually only one source)
        :param str i_str: name of input file or other input
        :param str t_path: location of intermediate chunk on disk
        :return: list of output file paths or other outputs


def _run_incremental_transforms(self, si, transforms):

<EOL>for transform in transforms:<EOL><INDENT>try:<EOL><INDENT>stream_id = si.stream_id<EOL>si_new = transform(si, context=self.context)<EOL>if si_new is None:<EOL><INDENT>logger.warn('<STR_LIT>',<EOL>transform, stream_id, si and si.abs_url)<EOL>return None<EOL><DEDENT>si = si_new<EOL><DEDENT>except TransformGivingUp:<EOL><INDENT>logger.info('<STR_LIT>',<EOL>transform, si.stream_id)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.critical(<EOL>'<STR_LIT>',<EOL>transform, si and si.stream_id, self.context.get('<STR_LIT>'),<EOL>si and si.abs_url, exc_info=True)<EOL><DEDENT><DEDENT>assert si is not None<EOL>if not si.stream_time:<EOL><INDENT>raise InvalidStreamItem('<STR_LIT>' % si)<EOL><DEDENT>if si.stream_id is None:<EOL><INDENT>raise InvalidStreamItem('<STR_LIT>' % si)<EOL><DEDENT>if type(si) != streamcorpus.StreamItem_v0_3_0:<EOL><INDENT>raise InvalidStreamItem('<STR_LIT>' %<EOL>type(si))<EOL><DEDENT>self.t_chunk.add(si)<EOL>return si<EOL>

Run transforms on stream item.
Item may be discarded by some transform.
Writes successful items out to current self.t_chunk
Returns transformed item or None.


return re.sub(r'<STR_LIT>', '<STR_LIT:U+0020>',<EOL>possibly_invalid_string)<EOL>

Clean unexpected Unicode characters, including non-BMP.

    Returns a copy of `possibly_invalid_string` with the following
    replaced by space (U+0020): ASCII control characters other than
    tab, carriage return, and newline; reserved code points for
    surrogate pairs; invalid characters U+FFFE and U+FFFF; and
    supplementary characters U+10000 and higher.

    :param unicode possibly_invalid_string: string to clean
    :return: cleaned string
    :returntype: :class:`unicode`


converted = UnicodeDammit(raw, isHTML=True)<EOL>if not converted.str:<EOL><INDENT>converted.str = str(raw, '<STR_LIT:utf8>', errors='<STR_LIT:ignore>')<EOL><DEDENT>encoding_m = encoding_re.match(converted.str)<EOL>if encoding_m:<EOL><INDENT>converted.str =encoding_m.group('<STR_LIT>') +encoding_m.group('<STR_LIT>')<EOL><DEDENT>return converted.str<EOL>

Try really really hard to get a Unicode copy of a string.

    First try :class:`BeautifulSoup.UnicodeDammit` to try to force
    to Unicode; if that fails, assume UTF-8 encoding, and ignore
    all errors.

    :param str raw: string to coerce
    :return: Unicode approximation of `raw`
    :returntype: :class:`unicode`


def make_clean_html(raw, stream_item=None, encoding=None):

<EOL>raw = fix_emails(raw)<EOL>raw_decoded = nice_decode(raw, stream_item=stream_item, encoding=encoding)<EOL>if raw_decoded is None:<EOL><INDENT>raw_decoded = raw<EOL><DEDENT>try:<EOL><INDENT>root = lxml.html.document_fromstring(raw_decoded)<EOL><DEDENT>except ValueError as exc:<EOL><INDENT>if '<STR_LIT>' in str(exc):<EOL><INDENT>root = lxml.html.document_fromstring(raw)<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>lxml.etree.strip_attributes(root, '<STR_LIT:class>', '<STR_LIT:id>')<EOL>fixed_html = lxml.html.tostring(root, encoding=str)<EOL>fixed_html = string.replace(fixed_html, '<STR_LIT:\r>', '<STR_LIT:U+0020>')<EOL>fixed_html = drop_invalid_and_upper_utf8_chars(fixed_html)<EOL>cleaner = lxml.html.clean.Cleaner(<EOL>scripts=True, javascript=True,<EOL>comments=True,<EOL>page_structure=False,<EOL>remove_tags=['<STR_LIT>'],<EOL>style=True, links=True)<EOL>_clean_html = cleaner.clean_html(fixed_html)<EOL>_clean_html = lxml.html.tostring(<EOL>lxml.html.document_fromstring(_clean_html),<EOL>method='<STR_LIT:html>', encoding='<STR_LIT:utf-8>',<EOL>pretty_print=True,<EOL>)<EOL>return uniform_html(_clean_html)<EOL>

Get a clean text representation of presumed HTML.

    Treat `raw` as though it is HTML, even if we have no idea what it
    really is, and attempt to get a properly formatted HTML document
    with all HTML-escaped characters converted to their unicode.

    This is called below by the `clean_html` transform stage, which
    interprets MIME-type.  If `character_encoding` is not provided,
    and `stream_item` is provided, then this falles back to
    :attr:`streamcorpus.StreamItem.body.encoding`.

    :param str raw: raw text to clean up
    :param stream_item: optional stream item with encoding metadata
    :type stream_item: :class:`streamcorpus.StreamItem`
    :returns: UTF-8-encoded byte string of cleaned HTML text
    :returntype: :class:`str`


doc = html5lib.parse(html.decode('<STR_LIT:utf-8>'))<EOL>config = {<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': '<STR_LIT:utf-8>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>}<EOL>return html5lib.serializer.serialize(doc, **config)<EOL>

Takes a utf-8-encoded string of HTML as input and returns a new
    HTML string with fixed quoting and close tags, which generally
    should not break any of the offsets and makes it easier for
    functions like
    :func:`streamcorpus_pipeline.offsets.char_offsets_to_xpaths` to
    operate without failures.


if len(self.include_mime_types) == <NUM_LIT:0>:<EOL><INDENT>return True<EOL><DEDENT>if mime_type is None:<EOL><INDENT>return False<EOL><DEDENT>mime_type = mime_type.lower()<EOL>return any(mime_type.startswith(mt) for mt in self.include_mime_types)<EOL>

This implements the MIME-type matching logic for deciding whether
        to run `make_clean_html`


def rmtree(path, use_shutil=True, followlinks=False, retries=<NUM_LIT:10>):

if use_shutil and not followlinks:<EOL><INDENT>try:<EOL><INDENT>shutil.rmtree(path)<EOL>return<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.info('<STR_LIT>', path)<EOL>logger.debug('<STR_LIT>', traceback.format_exc(exc))<EOL><DEDENT><DEDENT>if not os.path.isdir(path):<EOL><INDENT>os.remove(path)<EOL>return<EOL><DEDENT>for root, dir_names, file_names in os.walk(path, topdown=False, followlinks=followlinks):<EOL><INDENT>for fname in file_names:<EOL><INDENT>fpath = os.path.join(root, fname)<EOL>tries = <NUM_LIT:0><EOL>while tries < retries:<EOL><INDENT>tries += <NUM_LIT:1><EOL>try:<EOL><INDENT>os.remove(fpath)<EOL>break<EOL><DEDENT>except Exception as exc:<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>if os.path.exists(fpath):<EOL><INDENT>logger.critical('<STR_LIT>', fpath)<EOL>logger.critical('<STR_LIT>', traceback.format_exc(exc))<EOL><DEDENT><DEDENT>for dname in dir_names:<EOL><INDENT>full_path = os.path.join(root, dname)<EOL>if os.path.islink(full_path):<EOL><INDENT>real_path = os.path.realpath(full_path)<EOL>os.remove(full_path)<EOL>full_path = real_path<EOL><DEDENT>os.rmdir(full_path)<EOL><DEDENT><DEDENT>if os.path.exists(path):<EOL><INDENT>os.rmdir(path)<EOL><DEDENT>

remove all files and directories below path, including path
    itself; works even when shutil.rmtree fails because of read-only
    files in NFS and Windows.  Follows symlinks.

    `use_shutil` defaults to True; useful for testing

    `followlinks` defaults to False; if set to True, shutil.rmtree is
    not used.


pid = os.getpid()<EOL>procs = subprocess.check_output( <EOL>[ "<STR_LIT>", '<STR_LIT>', '<STR_LIT>', "<STR_LIT>", str( pid ) ] )<EOL>if verbose:<EOL><INDENT>oprocs = subprocess.check_output( <EOL>[ "<STR_LIT>", '<STR_LIT>', "<STR_LIT>", str( pid ) ] )<EOL>logger.info(oprocs)<EOL><DEDENT>open_files = [s for s in procs.split( '<STR_LIT:\n>' ) if s and s[ <NUM_LIT:0> ] == '<STR_LIT:f>' and s[<NUM_LIT:1>: ].isdigit()]<EOL>return open_files<EOL>

return list of open files for current process

    .. warning: will only work on UNIX-like os-es.


def parse_keys_and_ranges(i_str, keyfunc, rangefunc):

while i_str:<EOL><INDENT>m = _STREAM_ID_RE.match(i_str)<EOL>if m:<EOL><INDENT>for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):<EOL><INDENT>yield retval<EOL><DEDENT>i_str = i_str[m.end():]<EOL>while i_str and ((i_str[<NUM_LIT:0>] == '<STR_LIT:U+002C>') or (i_str[<NUM_LIT:0>] == '<STR_LIT:;>')):<EOL><INDENT>i_str = i_str[<NUM_LIT:1>:]<EOL><DEDENT>continue<EOL><DEDENT>if len(i_str) == SI_KEY_LENGTH:<EOL><INDENT>key = parse_si_key(i_str)<EOL>for retval in keyfunc(key):<EOL><INDENT>yield retval<EOL><DEDENT>return<EOL><DEDENT>keya = i_str[:SI_KEY_LENGTH]<EOL>splitc = i_str[SI_KEY_LENGTH]<EOL>if splitc == '<STR_LIT:<>':<EOL><INDENT>keyb = i_str[SI_KEY_LENGTH+<NUM_LIT:1>:SI_KEY_LENGTH+<NUM_LIT:1>+SI_KEY_LENGTH]<EOL>i_str = i_str[SI_KEY_LENGTH+<NUM_LIT:1>+SI_KEY_LENGTH:]<EOL>keya = parse_si_key(keya)<EOL>keyb = parse_si_key(keyb)<EOL>for retval in rangefunc(keya, keyb):<EOL><INDENT>yield retval<EOL><DEDENT><DEDENT>elif splitc == '<STR_LIT:;>':<EOL><INDENT>keya = parse_si_key(keya)<EOL>for retval in keyfunc(keya):<EOL><INDENT>yield retval<EOL><DEDENT>i_str = i_str[SI_KEY_LENGTH+<NUM_LIT:1>+<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>', splitc, i_str)<EOL>return<EOL><DEDENT><DEDENT>

Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.


if client is None:<EOL><INDENT>client = kvlayer.client()<EOL>client.setup_namespace(STREAM_ITEM_TABLE_DEFS,<EOL>STREAM_ITEM_VALUE_DEFS)<EOL><DEDENT>key = stream_id_to_kvlayer_key(stream_id)<EOL>for k, v in client.get(STREAM_ITEMS_TABLE, key):<EOL><INDENT>if v is not None:<EOL><INDENT>errors, bytestr = streamcorpus.decrypt_and_uncompress(v)<EOL>return streamcorpus.deserialize(bytestr)<EOL><DEDENT><DEDENT>raise KeyError(stream_id)<EOL>

Retrieve a :class:`streamcorpus.StreamItem` from :mod:`kvlayer`.

    This function requires that `client` already be set up properly::

        client = kvlayer.client()
        client.setup_namespace(STREAM_ITEM_TABLE_DEFS,
                               STREAM_ITEM_VALUE_DEFS)
        si = get_kvlayer_stream_item(client, stream_id)

    `stream_id` is in the form of
    :data:`streamcorpus.StreamItem.stream_id` and contains the
    ``epoch_ticks``, a hyphen, and the ``doc_id``.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str stream_id: stream Id to retrieve
    :return: corresponding :class:`streamcorpus.StreamItem`
    :raise exceptions.KeyError: if `stream_id` is malformed or does
      not correspond to anything in the database


assert len(doc_id) == <NUM_LIT:32>, '<STR_LIT>' % doc_id<EOL>bin_docid = base64.b16decode(doc_id.upper())<EOL>doc_id_range = ((bin_docid,), (bin_docid,))<EOL>return doc_id_range<EOL>

Construct a tuple(begin, end) of one-tuple kvlayer keys from a
    hexdigest doc_id.


if client is None:<EOL><INDENT>client = kvlayer.client()<EOL>client.setup_namespace(STREAM_ITEM_TABLE_DEFS,<EOL>STREAM_ITEM_VALUE_DEFS)<EOL><DEDENT>doc_id_range = make_doc_id_range(doc_id)<EOL>for k, v in client.scan(STREAM_ITEMS_TABLE, doc_id_range):<EOL><INDENT>if v is not None:<EOL><INDENT>errors, bytestr = streamcorpus.decrypt_and_uncompress(v)<EOL>yield streamcorpus.deserialize(bytestr)<EOL><DEDENT><DEDENT>

Retrieve :class:`streamcorpus.StreamItem`s from :mod:`kvlayer`.

    Namely, it returns an iterator over all documents with the given
    docid. The docid should be an md5 hash of the document's abs_url.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str doc_id: doc id of documents to retrieve
    :return: generator of :class:`streamcorpus.StreamItem`


if client is None:<EOL><INDENT>client = kvlayer.client()<EOL>client.setup_namespace(STREAM_ITEM_TABLE_DEFS,<EOL>STREAM_ITEM_VALUE_DEFS)<EOL><DEDENT>doc_id_range = make_doc_id_range(doc_id)<EOL>for k in client.scan_keys(STREAM_ITEMS_TABLE, doc_id_range):<EOL><INDENT>yield kvlayer_key_to_stream_id(k)<EOL><DEDENT>

Retrieve stream ids from :mod:`kvlayer`.

    Namely, it returns an iterator over all stream ids with the given
    docid. The docid should be an md5 hash of the document's abs_url.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str doc_id: doc id of documents to retrieve
    :return: generator of str


if len(si_key[<NUM_LIT:0>]) != <NUM_LIT:16>:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>'.format(si_key))<EOL><DEDENT>return struct.pack('<STR_LIT>', si_key[<NUM_LIT:0>], si_key[<NUM_LIT:1>])<EOL>

Return packed bytes representation of StreamItem kvlayer key.
The result is 20 bytes, 16 of md5 hash, 4 of int timestamp.


return struct.unpack('<STR_LIT>', si_key_bytes)<EOL>

Parse result of serialize_si_key()
Return kvlayer key tuple.


key = key_for_stream_item(si)<EOL>data = streamcorpus.serialize(si)<EOL>errors, data = streamcorpus.compress_and_encrypt(data)<EOL>assert not errors, errors<EOL>return key, data<EOL>

extract the parts of a StreamItem that go into a kvlayer key,
convert StreamItem to blob for storage.

return (kvlayer key tuple), data blob


if si.source:<EOL><INDENT>yield (STREAM_ITEMS_SOURCE_INDEX, (key_for_stream_item(si), si.source))<EOL><DEDENT>


if not i_str:<EOL><INDENT>for si in self._loadrange(None, None):<EOL><INDENT>yield si<EOL><DEDENT>return<EOL><DEDENT>for si in parse_keys_and_ranges(i_str, self._loadkey,<EOL>self._loadrange):<EOL><INDENT>yield si<EOL><DEDENT>

Actually scan the table, yielding stream items.

        `i_str` consists of a sequence of 20-byte encoded stream IDs.
        These are the 16 bytes of the document ID, plus the 4-byte
        big-endian stream time.  :func:`serialize_si` can produce
        these.  `i_str` can then consist of a single encoded stream
        ID; an encoded stream ID, a literal ``<``, and a second
        encoded stream ID to specify a range of documents; or a list
        of either of the preceding separated by literal ``;``.

        .. todo:: make this support keyword index strings


global _proc_status, _scale<EOL>try:<EOL><INDENT>t = open(_proc_status)<EOL>v = t.read()<EOL>t.close()<EOL><DEDENT>except:<EOL><INDENT>return <NUM_LIT:0.0>  <EOL><DEDENT>i = v.index(VmKey)<EOL>v = v[i:].split(None, <NUM_LIT:3>)  <EOL>if len(v) < <NUM_LIT:3>:<EOL><INDENT>return <NUM_LIT:0.0>  <EOL><DEDENT>return float(v[<NUM_LIT:1>]) * _scale[v[<NUM_LIT:2>]]<EOL>


return _VmB('<STR_LIT>') - since<EOL>


tmp_cleansed = open(tmp_cleansed_path, '<STR_LIT:wb>')<EOL>for idx, si in enumerate(i_chunk):<EOL><INDENT>tmp_cleansed.write('<STR_LIT>' % si.stream_id)<EOL>tmp_cleansed.write(si.body.cleansed)<EOL>tmp_cleansed.write('<STR_LIT>')<EOL><DEDENT>tmp_cleansed.close()<EOL>print('<STR_LIT>' % tmp_cleansed_path)<EOL>


def make_ner_file(tagger_id, tmp_cleansed_path, tmp_ner_path, pipeline_root):

params = dict(INPUT_FILE=tmp_cleansed_path,<EOL>OUTPUT_FILE=tmp_ner_path,<EOL>PIPELINE_ROOT=pipeline_root)<EOL>pipeline_cmd = pipeline_cmd_templates[tagger_id] % params<EOL>print(pipeline_cmd)<EOL>print('<STR_LIT>' % tmp_ner_path)<EOL>start_time = time.time()<EOL>gpg_child = subprocess.Popen(<EOL>pipeline_cmd,<EOL>stderr=subprocess.PIPE, shell=True)<EOL>s_out, errors = gpg_child.communicate()<EOL>assert gpg_child.returncode == <NUM_LIT:0> and '<STR_LIT>' not in errors, errors<EOL>elapsed = time.time() - start_time<EOL>print('<STR_LIT>' % (tmp_ner_path, elapsed))<EOL>'''<STR_LIT>'''<EOL>


try:<EOL><INDENT>span = span.encode('<STR_LIT:utf8>', '<STR_LIT:ignore>')<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>span = span.lower()<EOL>span = span.translate(strip_punctuation)<EOL>span = whitespace.sub('<STR_LIT:U+0020>', span)<EOL>return span.strip()<EOL>

Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string


def align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path):

o_chunk = Chunk()<EOL>input_iter = i_chunk.__iter__()<EOL>ner = '<STR_LIT>'<EOL>stream_id = None<EOL>all_ner = xml.dom.minidom.parse(open(tmp_ner_path))<EOL>for raw_ner in all_ner.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>stream_item = next(input_iter)<EOL>stream_id = raw_ner.attributes.get('<STR_LIT>').value<EOL>assert stream_id and stream_id == stream_item.stream_id,'<STR_LIT>' % (stream_id, stream_item.stream_id, ner)<EOL>tagger_id = '<STR_LIT>'<EOL>tagging = Tagging()<EOL>tagging.tagger_id = tagger_id<EOL>tagged_doc = list(lingpipe.files(raw_ner.toxml()))[<NUM_LIT:0>][<NUM_LIT:1>]<EOL>tagging.raw_tagging = tagged_doc<EOL>tagging.generation_time = streamcorpus.make_stream_time()<EOL>stream_item.body.taggings[tagger_id] = tagging<EOL>sentences = list(lingpipe.sentences(tagged_doc))<EOL>assert stream_item.ratings[<NUM_LIT:0>].mentions, stream_item.stream_id<EOL>john_smith_label = Label()<EOL>john_smith_label.annotator = stream_item.ratings[<NUM_LIT:0>].annotator<EOL>john_smith_label.target_id = stream_item.ratings[<NUM_LIT:0>].target_id<EOL>equiv_ids = collections.defaultdict(lambda: set())<EOL>for sent in sentences:<EOL><INDENT>for tok in sent.tokens:<EOL><INDENT>if tok.entity_type is not None:<EOL><INDENT>equiv_ids[tok.equiv_id].add(cleanse(tok.token))<EOL><DEDENT><DEDENT><DEDENT>johnsmiths = set()<EOL>for equiv_id, names in list(equiv_ids.items()):<EOL><INDENT>_names = cleanse('<STR_LIT:U+0020>'.join(names))<EOL>if '<STR_LIT>' in _names and '<STR_LIT>' in _names:<EOL><INDENT>johnsmiths.add(equiv_id)<EOL><DEDENT><DEDENT>print(len(johnsmiths))<EOL>for sent in sentences:<EOL><INDENT>for tok in sent.tokens:<EOL><INDENT>if tok.equiv_id in johnsmiths:<EOL><INDENT>tok.labels = [john_smith_label]                <EOL><DEDENT><DEDENT><DEDENT>stream_item.body.sentences[tagger_id] = sentences<EOL>o_chunk.add(stream_item)<EOL><DEDENT>open(tmp_done_path, '<STR_LIT:wb>').write(str(o_chunk))<EOL>print('<STR_LIT>' % tmp_done_path)<EOL>

iterate through the i_chunk and tmp_ner_path to generate a new
Chunk with body.ner


<EOL>fname = work_unit.key.strip().split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>output_dir_path = work_unit.data.get('<STR_LIT>', '<STR_LIT>')<EOL>u = uuid.uuid3(uuid.UUID(int=<NUM_LIT:0>), work_unit.key.strip())<EOL>path1 = u.hex[<NUM_LIT:0>]<EOL>path2 = u.hex[<NUM_LIT:1>]<EOL>fpath = os.path.join(output_dir_path, path1, path2, fname)<EOL>if not os.path.exists(os.path.dirname(fpath)):<EOL><INDENT>os.makedirs(os.path.dirname(fpath))<EOL><DEDENT>output = gzip.open(fpath + '<STR_LIT>', '<STR_LIT:wb>')<EOL>expected_si_count = int(fname.split('<STR_LIT:->')[<NUM_LIT:1>])<EOL>max_tries = <NUM_LIT:20><EOL>tries = <NUM_LIT:0><EOL>while tries < max_tries:<EOL><INDENT>try:<EOL><INDENT>exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids =attempt_fetch(work_unit, fpath)<EOL>if si_count != expected_si_count:<EOL><INDENT>print('<STR_LIT>' % (si_count, expected_si_count))<EOL>sys.stdout.flush()<EOL>tries += <NUM_LIT:1><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' % (si_count,))<EOL>sys.stdout.flush()<EOL><DEDENT>output.write( '<STR_LIT>' % (<EOL>exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, <EOL>work_unit.key.strip(), '<STR_LIT:U+002C>'.join(['<STR_LIT>' % tup for tup in stream_ids])) )<EOL>break<EOL><DEDENT>except Exception as exc:<EOL><INDENT>print('<STR_LIT>')<EOL>print(traceback.format_exc(exc))<EOL>sys.stdout.flush()<EOL>tries += <NUM_LIT:1><EOL>output.write(traceback.format_exc(exc))<EOL><DEDENT><DEDENT>output.close()<EOL>

get a rejester.WorkUnit with KBA s3 path, fetch it, and save
    some counts about it.


url = '<STR_LIT>' + work_unit.key.strip()<EOL>cmd = '<STR_LIT>' % (url, fpath)<EOL>print(cmd)<EOL>child = Popen(cmd, stdout=PIPE, shell=True)<EOL>print('<STR_LIT>')<EOL>sys.stdout.flush()<EOL>si_count = <NUM_LIT:0><EOL>serif_count = <NUM_LIT:0><EOL>exc = '<STR_LIT>'<EOL>stream_ids = list()<EOL>clean_visible_bytes = <NUM_LIT:0><EOL>clean_visible_count = <NUM_LIT:0><EOL>try:<EOL><INDENT>for si in Chunk(file_obj=child.stdout):<EOL><INDENT>print(si.stream_id, si.abs_url)<EOL>if si.body.language:<EOL><INDENT>lang = si.body.language.code<EOL><DEDENT>else:<EOL><INDENT>lang = '<STR_LIT>'<EOL><DEDENT>stream_ids.append((lang, si.stream_id))<EOL>if si.body.clean_visible:<EOL><INDENT>clean_visible_count += <NUM_LIT:1><EOL>clean_visible_bytes += len(si.body.clean_visible)<EOL><DEDENT>si_count += <NUM_LIT:1><EOL>if '<STR_LIT>' in si.body.sentences:<EOL><INDENT>serif_count += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>exc = re.sub('<STR_LIT>', '<STR_LIT:U+0020>', str(exc)).strip()<EOL><DEDENT>child.terminate()<EOL>child.wait()<EOL>child.stdout.close()<EOL>return exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids<EOL>

attempt a fetch and iteration over a work_unit.key path in s3


pass<EOL>


for root, dirs, fnames in os.walk(input_dir):<EOL><INDENT>for i_fname in fnames:<EOL><INDENT>i_path = os.path.join(root, i_fname)<EOL>yield i_path<EOL><DEDENT><DEDENT>


return white_space_re.sub("<STR_LIT:U+0020>", s.translate(strip_punctuation).lower())<EOL>

strips punctuation and repeated whitespace from unicode strings


if self.matcher.match(strip_string(si.body.clean_visible.decode('<STR_LIT:utf-8>'))):<EOL><INDENT>return si<EOL><DEDENT>


if not path_to_original.startswith('<STR_LIT:/>'):<EOL><INDENT>path_to_original = os.path.join(os.getcwd(), path_to_original)<EOL><DEDENT>for root_dir, dirnames, filenames in os.walk(path_to_original):<EOL><INDENT>for fname in fnmatch.filter(filenames, '<STR_LIT:*>'):<EOL><INDENT>dir_path = os.path.join(path_to_original, root_dir)<EOL>yield self._make_stream_item(dir_path, fname)<EOL><DEDENT><DEDENT>

This _looks_ like a Chunk only in that it generates StreamItem
instances when iterated upon.


def recursive_glob_with_tree(new_base, old_base, treeroot, pattern):

results = []<EOL>old_cwd = os.getcwd()<EOL>os.chdir(old_base)<EOL>for rel_base, dirs, files in os.walk(treeroot):<EOL><INDENT>goodfiles = fnmatch.filter(files, pattern)<EOL>one_dir_results = []<EOL>for f in goodfiles:<EOL><INDENT>one_dir_results.append(os.path.join(old_base, rel_base, f))<EOL><DEDENT>results.append((os.path.join(new_base, rel_base), one_dir_results))<EOL><DEDENT>os.chdir(old_cwd)<EOL>return results<EOL>

generate a list of tuples(new_base, list(paths to put there)
    where the files are found inside of old_base/treeroot.


def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,<EOL>to_dir=os.curdir, delay=<NUM_LIT:15>):

<EOL>to_dir = os.path.abspath(to_dir)<EOL>try:<EOL><INDENT>from urllib.request import urlopen<EOL><DEDENT>except ImportError:<EOL><INDENT>from urllib2 import urlopen<EOL><DEDENT>tgz_name = "<STR_LIT>" % version<EOL>url = download_base + tgz_name<EOL>saveto = os.path.join(to_dir, tgz_name)<EOL>src = dst = None<EOL>if not os.path.exists(saveto):  <EOL><INDENT>try:<EOL><INDENT>log.warn("<STR_LIT>", url)<EOL>src = urlopen(url)<EOL>data = src.read()<EOL>dst = open(saveto, "<STR_LIT:wb>")<EOL>dst.write(data)<EOL><DEDENT>finally:<EOL><INDENT>if src:<EOL><INDENT>src.close()<EOL><DEDENT>if dst:<EOL><INDENT>dst.close()<EOL><DEDENT><DEDENT><DEDENT>return os.path.realpath(saveto)<EOL>

Download distribute from a specified location and return its filename

    `version` should be a valid distribute version number that is available
    as an egg for download under the `download_base` URL (which should end
    with a '/'). `to_dir` is the directory where the egg will be downloaded.
    `delay` is the number of seconds to pause before an actual download
    attempt.


existing_content = open(path).read()<EOL>if existing_content == content:<EOL><INDENT>log.warn('<STR_LIT>')<EOL>return False<EOL><DEDENT>log.warn('<STR_LIT>')<EOL>_rename_path(path)<EOL>f = open(path, '<STR_LIT:w>')<EOL>try:<EOL><INDENT>f.write(content)<EOL><DEDENT>finally:<EOL><INDENT>f.close()<EOL><DEDENT>return True<EOL>


def _extractall(self, path="<STR_LIT:.>", members=None):

import copy<EOL>import operator<EOL>from tarfile import ExtractError<EOL>directories = []<EOL>if members is None:<EOL><INDENT>members = self<EOL><DEDENT>for tarinfo in members:<EOL><INDENT>if tarinfo.isdir():<EOL><INDENT>directories.append(tarinfo)<EOL>tarinfo = copy.copy(tarinfo)<EOL>tarinfo.mode = <NUM_LIT>  <EOL><DEDENT>self.extract(tarinfo, path)<EOL><DEDENT>if sys.version_info < (<NUM_LIT:2>, <NUM_LIT:4>):<EOL><INDENT>def sorter(dir1, dir2):<EOL><INDENT>return cmp(dir1.name, dir2.name)<EOL><DEDENT>directories.sort(sorter)<EOL>directories.reverse()<EOL><DEDENT>else:<EOL><INDENT>directories.sort(key=operator.attrgetter('<STR_LIT:name>'), reverse=True)<EOL><DEDENT>for tarinfo in directories:<EOL><INDENT>dirpath = os.path.join(path, tarinfo.name)<EOL>try:<EOL><INDENT>self.chown(tarinfo, dirpath)<EOL>self.utime(tarinfo, dirpath)<EOL>self.chmod(tarinfo, dirpath)<EOL><DEDENT>except ExtractError:<EOL><INDENT>e = sys.exc_info()[<NUM_LIT:1>]<EOL>if self.errorlevel > <NUM_LIT:1>:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>self._dbg(<NUM_LIT:1>, "<STR_LIT>" % e)<EOL><DEDENT><DEDENT><DEDENT>

Extract all members from the archive to the current working
       directory and set owner, modification time and permissions on
       directories afterwards. `path' specifies a different directory
       to extract to. `members' is optional and must be a subset of the
       list returned by getmembers().


install_args = []<EOL>if options.user_install:<EOL><INDENT>if sys.version_info < (<NUM_LIT:2>, <NUM_LIT:6>):<EOL><INDENT>log.warn("<STR_LIT>")<EOL>raise SystemExit(<NUM_LIT:1>)<EOL><DEDENT>install_args.append('<STR_LIT>')<EOL><DEDENT>return install_args<EOL>

Build the arguments to 'python setup.py install' on the distribute package


parser = optparse.OptionParser()<EOL>parser.add_option(<EOL>'<STR_LIT>', dest='<STR_LIT>', action='<STR_LIT:store_true>', default=False,<EOL>help='<STR_LIT>')<EOL>parser.add_option(<EOL>'<STR_LIT>', dest='<STR_LIT>', metavar="<STR_LIT>",<EOL>default=DEFAULT_URL,<EOL>help='<STR_LIT>')<EOL>options, args = parser.parse_args()<EOL>return options<EOL>


options = _parse_args()<EOL>tarball = download_setuptools(download_base=options.download_base)<EOL>return _install(tarball, _build_install_args(options))<EOL>

Install or upgrade setuptools and EasyInstall


Called by the dashboard when the display should try and draw the notification.

Implementations should block the thread whilst the display is showing a notification, as soon as the thread is
unblocked then all data-sources will be polled.

:param notification: The notification to draw that is of a type returned by `get_supported_notifications()`


This method is called when the consumer of the display wants to know what notification it supports.
:return: An array of all the notification types that are supported by the display.


@cli.command()<EOL>@click.argument("<STR_LIT>", type=click.Path(), nargs=-<NUM_LIT:1>)<EOL>@click.option('<STR_LIT>', is_flag=True, help='<STR_LIT>')<EOL>@click.option("<STR_LIT>", type=click.Path(exists=True))<EOL>@click.option("<STR_LIT>", is_flag=True, callback=attach_logging, expose_value=False)<EOL>def start(dashboards, once, secrets):

if secrets is None:<EOL><INDENT>secrets = os.path.join(os.path.expanduser("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>loaded_secrets = try_read_secrets_file(secrets)<EOL><DEDENT>except InvalidSecretsException as err:<EOL><INDENT>click.echo(get_error_message(err, default="<STR_LIT>"), err=True)<EOL>raise click.Abort()<EOL><DEDENT>read_configs = ["""<STR_LIT>"""]<EOL>for dashboard_file in dashboards:<EOL><INDENT>read_configs.append(read_file(dashboard_file))<EOL><DEDENT>dashboard_config = DashboardConfigReader(initialise_component_loader(), loaded_secrets)<EOL>try:<EOL><INDENT>dashboard = read_dashboard_from_config(dashboard_config, read_configs)<EOL><DEDENT>except YAMLError as err:<EOL><INDENT>click.echo(get_error_message(err, default="<STR_LIT>"), err=True)<EOL>raise click.Abort()<EOL><DEDENT>try:<EOL><INDENT>DashboardValidator().validate(dashboard)<EOL><DEDENT>except ValidationException as err:<EOL><INDENT>click.echo(get_error_message(err, default="<STR_LIT>"), err=True)<EOL>raise click.Abort()<EOL><DEDENT>explain_dashboard(dashboard)<EOL>click.echo("<STR_LIT>")<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>DashboardRunner(dashboard).cycle()<EOL><DEDENT>except SecretNotFound as err:<EOL><INDENT>click.echo(get_error_message(err, default="<STR_LIT>"), err=True)<EOL>raise click.Abort()<EOL><DEDENT>if once:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Display a dashboard from the dashboard file(s) provided in the DASHBOARDS
       Paths and/or URLs for dashboards (URLs must secrets with http or https)


@cli.command()<EOL>@click.argument("<STR_LIT:action>", type=click.Choice(["<STR_LIT>", "<STR_LIT>"]))<EOL>@click.argument("<STR_LIT>", type=click.Path(), nargs=-<NUM_LIT:1>)<EOL>@click.option("<STR_LIT>", type=click.Path(exists=True))<EOL>@click.option("<STR_LIT>", is_flag=True, callback=attach_logging, expose_value=False)<EOL>def view(action, dashboards, secrets):

if secrets is None:<EOL><INDENT>secrets = os.path.join(os.path.expanduser("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>loaded_secrets = try_read_secrets_file(secrets)<EOL><DEDENT>except InvalidSecretsException as err:<EOL><INDENT>click.echo(get_error_message(err, default="<STR_LIT>"), err=True)<EOL>raise click.Abort()<EOL><DEDENT>dashboard_config = DashboardConfigReader(initialise_component_loader(), loaded_secrets)<EOL>read_configs = [read_file(f) for f in dashboards]<EOL>dashboard = read_dashboard_from_config(dashboard_config, read_configs)<EOL>try:<EOL><INDENT>messages = DashboardRunner(dashboard).poll_datafeeds()<EOL><DEDENT>except SecretNotFound as err:<EOL><INDENT>click.echo(get_error_message(err, default="<STR_LIT>"), err=True)<EOL>raise click.Abort()<EOL><DEDENT>cli_output = {"<STR_LIT>": messages}<EOL>if action == "<STR_LIT>":<EOL><INDENT>cli_output["<STR_LIT>"] = []<EOL>for notification in dashboard.notifications:<EOL><INDENT>notification_output = notification.create(messages)<EOL>filtered_messages = messages<EOL>if isinstance(notification, FilteredNotification):<EOL><INDENT>filtered_messages = notification.filter_messages(messages)<EOL><DEDENT>cli_output["<STR_LIT>"].append({<EOL>"<STR_LIT>": filtered_messages,<EOL>"<STR_LIT>": str(notification_output)<EOL>})<EOL><DEDENT><DEDENT>json_output = json.dumps(cli_output, sort_keys=True, indent=<NUM_LIT:4>, cls=MessageJsonEncoder)<EOL>click.echo(json_output)<EOL>

View the output of the datafeeds and/or notifications used in your DASHBOARDS


@cli.command()<EOL>@click.argument("<STR_LIT>",<EOL>type=click.Choice(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT:all>"]),<EOL>default="<STR_LIT:all>")<EOL>def list(component_type):

config_loader = initialise_component_loader()<EOL>component_types = sorted({<EOL>"<STR_LIT>": lambda: config_loader.load_by_type(ComponentType.DISPLAY),<EOL>"<STR_LIT>": lambda: config_loader.load_by_type(ComponentType.DATA_FEED),<EOL>"<STR_LIT>": lambda: config_loader.load_by_type(ComponentType.FILTER),<EOL>"<STR_LIT>": lambda: config_loader.load_by_type(ComponentType.NOTIFICATION)<EOL>}.items(), key=lambda t: t[<NUM_LIT:0>])<EOL>def print_ids(creators):<EOL><INDENT>ids = {c.id_key_value[<NUM_LIT:1>] if hasattr(c, "<STR_LIT>") else c.get_id() for c in creators}<EOL>for i in sorted(ids):<EOL><INDENT>click.echo("<STR_LIT>" % i)<EOL><DEDENT><DEDENT>for k, v in component_types:<EOL><INDENT>if component_type == k or component_type == "<STR_LIT:all>":<EOL><INDENT>click.echo("<STR_LIT>" % k)<EOL>print_ids(v())<EOL><DEDENT>if component_type == "<STR_LIT:all>":<EOL><INDENT>click.echo("<STR_LIT>")<EOL><DEDENT><DEDENT>

List components that are available on your machine


Creates an output based on the content of the messages.
If no output can be created from the messages, then None can be returned
:param messages: Latest messages from polling data-sources
:return: Output or None


:return: Return an array of the types of outputs this notification produces. This is used to check the
display that's been configured can handle all the types of outputs the notification could produce.


return urllib.parse.quote(full_url, safe="<STR_LIT>")<EOL>

Encode invalid characters in URL to provide, such as spaces.

This implements code from the following URLs
https://bugs.python.org/issue14826
https://hg.python.org/cpython/rev/ebd37273e0fe


:param message: Message to filter
:return: True if the message should be kept, otherwise false.


if "<STR_LIT:type>" not in config:<EOL><INDENT>raise InvalidConfigurationException("<STR_LIT>" % config)<EOL><DEDENT>component_type = config["<STR_LIT:type>"]<EOL>component_config = self._get_config_by_id(self._component_configs, component_type)<EOL>if not component_config:<EOL><INDENT>raise ComponentNotFoundForType(component_type)<EOL><DEDENT>options = config.get("<STR_LIT>", {})<EOL>component = self._parse_item(component_config, options, config)<EOL>component.name = config.get("<STR_LIT:name>", "<STR_LIT>")<EOL>return component<EOL>

Parses the section of configuration pertaining to a component
:param config: dict of specific config section
:return:


:return: Returns the ID used to represent the component in the config


:param options: Element's options
:return:


Returns an array of all the component configs


self._text = text<EOL>self._source_name = str(source) if source else "<STR_LIT>"<EOL>

:param text: Entity's text
:param source: Source of the entity, which is converted to a string via `str()` to produce the source name


messages = self.poll_datafeeds()<EOL>notifications = self.process_notifications(messages)<EOL>self.draw_notifications(notifications)<EOL>

Cycles through notifications with latest results from data feeds.


def make_signal(sr, duration, fmin='<STR_LIT>', fmax='<STR_LIT>'):

if fmin is None:<EOL><INDENT>fmin = <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>fmin = librosa.note_to_hz(fmin) / sr<EOL><DEDENT>if fmax is None:<EOL><INDENT>fmax = <NUM_LIT:0.5><EOL><DEDENT>else:<EOL><INDENT>fmax = librosa.note_to_hz(fmax) / sr<EOL><DEDENT>return np.sin(np.cumsum(<NUM_LIT:2> * np.pi * np.logspace(np.log10(fmin),<EOL>np.log10(fmax),<EOL>num=int(duration * sr))))<EOL>


<EOL>print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file, sr=<NUM_LIT>)<EOL>hop_length = <NUM_LIT><EOL>print('<STR_LIT>')<EOL>onsets = librosa.onset.onset_detect(y=y,<EOL>sr=sr,<EOL>hop_length=hop_length)<EOL>print("<STR_LIT>".format(onsets.shape[<NUM_LIT:0>]))<EOL>onset_times = librosa.frames_to_time(onsets,<EOL>sr=sr,<EOL>hop_length=hop_length)<EOL>print('<STR_LIT>', output_csv)<EOL>librosa.output.times_csv(output_csv, onset_times)<EOL>print('<STR_LIT>')<EOL>

Onset detection function

    :parameters:
      - input_file : str
          Path to input audio file (wav, mp3, m4a, flac, etc.)

      - output_file : str
          Path to save onset timestamps as a CSV file


Argparse function to get the program parameters


print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file)<EOL>print('<STR_LIT>')<EOL>y_harm = librosa.effects.harmonic(y)<EOL>print('<STR_LIT>')<EOL>tuning = librosa.estimate_tuning(y=y_harm, sr=sr)<EOL>print('<STR_LIT>'.format(<NUM_LIT:100> * tuning))<EOL>

Load an audio file and estimate tuning (in cents)


parser = argparse.ArgumentParser(description='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>',<EOL>action='<STR_LIT:store>',<EOL>help='<STR_LIT>')<EOL>return vars(parser.parse_args(args))<EOL>


def hpss_demo(input_file, output_harmonic, output_percussive):

<EOL>print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file)<EOL>print('<STR_LIT>')<EOL>y_harmonic, y_percussive = librosa.effects.hpss(y)<EOL>print('<STR_LIT>', output_harmonic)<EOL>librosa.output.write_wav(output_harmonic, y_harmonic, sr)<EOL>print('<STR_LIT>', output_percussive)<EOL>librosa.output.write_wav(output_percussive, y_percussive, sr)<EOL>

HPSS demo function.

    :parameters:
      - input_file : str
          path to input audio
      - output_harmonic : str
          path to save output harmonic (wav)
      - output_percussive : str
          path to save output harmonic (wav)


print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file)<EOL>print('<STR_LIT>')<EOL>y_harm = librosa.effects.harmonic(y)<EOL>print('<STR_LIT>')<EOL>tuning = librosa.estimate_tuning(y=y_harm, sr=sr)<EOL>print('<STR_LIT>'.format(<NUM_LIT:100> * tuning))<EOL>print('<STR_LIT>'.format(-<NUM_LIT:100> * tuning))<EOL>y_tuned = librosa.effects.pitch_shift(y, sr, -tuning)<EOL>print('<STR_LIT>', output_file)<EOL>librosa.output.write_wav(output_file, y_tuned, sr)<EOL>

Load audio, estimate tuning, apply pitch correction, and save.


<EOL>print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file)<EOL>print('<STR_LIT>')<EOL>y = librosa.effects.percussive(y)<EOL>print('<STR_LIT>')<EOL>onset_env = librosa.onset.onset_strength(y=y,<EOL>sr=sr,<EOL>hop_length=HOP_LENGTH,<EOL>n_fft=N_FFT,<EOL>aggregate=np.median)<EOL>tempo, beats = librosa.beat.beat_track(onset_envelope=onset_env,<EOL>sr=sr,<EOL>hop_length=HOP_LENGTH)<EOL>beat_times = librosa.frames_to_time(beats,<EOL>sr=sr,<EOL>hop_length=HOP_LENGTH)<EOL>print('<STR_LIT>', output_csv)<EOL>librosa.output.times_csv(output_csv, beat_times)<EOL>

HPSS beat tracking

    :parameters:
      - input_file : str
          Path to input audio file (wav, mp3, m4a, flac, etc.)

      - output_file : str
          Path to save beat event timestamps as a CSV file


def stretch_demo(input_file, output_file, speed):

<EOL>print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file)<EOL>print('<STR_LIT>'.format(speed * <NUM_LIT:100>))<EOL>y_stretch = librosa.effects.time_stretch(y, speed)<EOL>print('<STR_LIT>', output_file)<EOL>librosa.output.write_wav(output_file, y_stretch, sr)<EOL>

Phase-vocoder time stretch demo function.

    :parameters:
      - input_file : str
          path to input audio
      - output_file : str
          path to save output (wav)
      - speed : float > 0
          speed up by this factor


print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file, sr=<NUM_LIT>)<EOL>hop_length = <NUM_LIT><EOL>print('<STR_LIT>')<EOL>tempo, beats = librosa.beat.beat_track(y=y, sr=sr, hop_length=hop_length)<EOL>print('<STR_LIT>'.format(tempo))<EOL>beat_times = librosa.frames_to_time(beats, sr=sr, hop_length=hop_length)<EOL>print('<STR_LIT>', output_csv)<EOL>librosa.output.times_csv(output_csv, beat_times)<EOL>print('<STR_LIT>')<EOL>

Beat tracking function

    :parameters:
      - input_file : str
          Path to input audio file (wav, mp3, m4a, flac, etc.)

      - output_file : str
          Path to save beat event timestamps as a CSV file


def annotation(path, intervals, annotations=None, delimiter='<STR_LIT:U+002C>', fmt='<STR_LIT>'):

util.valid_intervals(intervals)<EOL>if annotations is not None and len(annotations) != len(intervals):<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>with open(path, '<STR_LIT:w>') as output_file:<EOL><INDENT>writer = csv.writer(output_file, delimiter=delimiter)<EOL>if annotations is None:<EOL><INDENT>for t_int in intervals:<EOL><INDENT>writer.writerow([fmt % t_int[<NUM_LIT:0>], fmt % t_int[<NUM_LIT:1>]])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for t_int, lab in zip(intervals, annotations):<EOL><INDENT>writer.writerow([fmt % t_int[<NUM_LIT:0>], fmt % t_int[<NUM_LIT:1>], lab])<EOL><DEDENT><DEDENT><DEDENT>

r'''Save annotations in a 3-column format::

        intervals[0, 0],intervals[0, 1],annotations[0]\n
        intervals[1, 0],intervals[1, 1],annotations[1]\n
        intervals[2, 0],intervals[2, 1],annotations[2]\n
        ...

    This can be used for segment or chord annotations.

    Parameters
    ----------
    path : str
        path to save the output CSV file

    intervals : np.ndarray [shape=(n, 2)]
        array of interval start and end-times.

        `intervals[i, 0]` marks the start time of interval `i`

        `intervals[i, 1]` marks the end time of interval `i`

    annotations : None or list-like [shape=(n,)]
        optional list of annotation strings. `annotations[i]` applies
        to the time range `intervals[i, 0]` to `intervals[i, 1]`

    delimiter : str
        character to separate fields

    fmt : str
        format-string for rendering time data

    Raises
    ------
    ParameterError
        if `annotations` is not `None` and length does
        not match `intervals`

    Examples
    --------
    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> data = librosa.feature.mfcc(y=y, sr=sr, hop_length=512)

    Detect segment boundaries

    >>> boundaries = librosa.segment.agglomerative(data, k=10)

    Convert to time

    >>> boundary_times = librosa.frames_to_time(boundaries, sr=sr,
    ...                                         hop_length=512)

    Convert events boundaries to intervals

    >>> intervals = np.hstack([boundary_times[:-1, np.newaxis],
    ...                        boundary_times[1:, np.newaxis]])

    Make some fake annotations

    >>> labels = ['Seg #{:03d}'.format(i) for i in range(len(intervals))]

    Save the output

    >>> librosa.output.annotation('segments.csv', intervals,
    ...                           annotations=labels)


def times_csv(path, times, annotations=None, delimiter='<STR_LIT:U+002C>', fmt='<STR_LIT>'):

if annotations is not None and len(annotations) != len(times):<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>with open(path, '<STR_LIT:w>') as output_file:<EOL><INDENT>writer = csv.writer(output_file, delimiter=delimiter)<EOL>if annotations is None:<EOL><INDENT>for t in times:<EOL><INDENT>writer.writerow([fmt % t])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for t, lab in zip(times, annotations):<EOL><INDENT>writer.writerow([(fmt % t), lab])<EOL><DEDENT><DEDENT><DEDENT>

r"""Save time steps as in CSV format.  This can be used to store the output
    of a beat-tracker or segmentation algorithm.

    If only `times` are provided, the file will contain each value
    of `times` on a row::

        times[0]\n
        times[1]\n
        times[2]\n
        ...

    If `annotations` are also provided, the file will contain
    delimiter-separated values::

        times[0],annotations[0]\n
        times[1],annotations[1]\n
        times[2],annotations[2]\n
        ...

    Parameters
    ----------
    path : string
        path to save the output CSV file

    times : list-like of floats
        list of frame numbers for beat events

    annotations : None or list-like
        optional annotations for each time step

    delimiter : str
        character to separate fields

    fmt : str
        format-string for rendering time

    Raises
    ------
    ParameterError
        if `annotations` is not `None` and length does not
        match `times`

    Examples
    --------
    Write beat-tracker time to CSV

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> tempo, beats = librosa.beat.beat_track(y, sr=sr, units='time')
    >>> librosa.output.times_csv('beat_times.csv', beats)


<EOL>util.valid_audio(y, mono=False)<EOL>if norm and np.issubdtype(y.dtype, np.floating):<EOL><INDENT>wav = util.normalize(y, norm=np.inf, axis=None)<EOL><DEDENT>else:<EOL><INDENT>wav = y<EOL><DEDENT>if wav.ndim > <NUM_LIT:1> and wav.shape[<NUM_LIT:0>] == <NUM_LIT:2>:<EOL><INDENT>wav = wav.T<EOL><DEDENT>scipy.io.wavfile.write(path, sr, wav)<EOL>

Output a time series as a .wav file

    Note: only mono or stereo, floating-point data is supported.
        For more advanced and flexible output options, refer to
        `soundfile`.

    Parameters
    ----------
    path : str
        path to save the output wav file

    y : np.ndarray [shape=(n,) or (2,n), dtype=np.float]
        audio time series (mono or stereo).

        Note that only floating-point values are supported.

    sr : int > 0 [scalar]
        sampling rate of `y`

    norm : boolean [scalar]
        enable amplitude normalization.
        For floating point `y`, scale the data to the range [-1, +1].

    Examples
    --------
    Trim a signal to 5 seconds and save it back

    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      duration=5.0)
    >>> librosa.output.write_wav('file_trim_5s.wav', y, sr)

    See Also
    --------
    soundfile.write


core_deps = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']<EOL>extra_deps = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>'.format(sys.version))<EOL>print('<STR_LIT>'.format(version))<EOL>for dep in core_deps:<EOL><INDENT>print('<STR_LIT>'.format(dep, __get_mod_version(dep)))<EOL><DEDENT>print('<STR_LIT>')<EOL>for dep in extra_deps:<EOL><INDENT>print('<STR_LIT>'.format(dep, __get_mod_version(dep)))<EOL><DEDENT>pass<EOL>

Return the version information for all librosa dependencies.


def onset_detect(y=None, sr=<NUM_LIT>, onset_envelope=None, hop_length=<NUM_LIT>,<EOL>backtrack=False, energy=None,<EOL>units='<STR_LIT>', **kwargs):

<EOL>if onset_envelope is None:<EOL><INDENT>if y is None:<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>onset_envelope = onset_strength(y=y, sr=sr, hop_length=hop_length)<EOL><DEDENT>onset_envelope -= onset_envelope.min()<EOL>if not onset_envelope.any():<EOL><INDENT>return np.array([], dtype=np.int)<EOL><DEDENT>onset_envelope /= onset_envelope.max()<EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length)       <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length + <NUM_LIT:1>)  <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length)       <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length + <NUM_LIT:1>)  <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length)          <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>)<EOL>onsets = util.peak_pick(onset_envelope, **kwargs)<EOL>if backtrack:<EOL><INDENT>if energy is None:<EOL><INDENT>energy = onset_envelope<EOL><DEDENT>onsets = onset_backtrack(onsets, energy)<EOL><DEDENT>if units == '<STR_LIT>':<EOL><INDENT>pass<EOL><DEDENT>elif units == '<STR_LIT>':<EOL><INDENT>onsets = core.frames_to_samples(onsets, hop_length=hop_length)<EOL><DEDENT>elif units == '<STR_LIT:time>':<EOL><INDENT>onsets = core.frames_to_time(onsets, hop_length=hop_length, sr=sr)<EOL><DEDENT>else:<EOL><INDENT>raise ParameterError('<STR_LIT>'.format(units))<EOL><DEDENT>return onsets<EOL>

Basic onset detector.  Locate note onset events by picking peaks in an
    onset strength envelope.

    The `peak_pick` parameters were chosen by large-scale hyper-parameter
    optimization over the dataset provided by [1]_.

    .. [1] https://github.com/CPJKU/onset_db

    Parameters
    ----------
    y          : np.ndarray [shape=(n,)]
        audio time series

    sr         : number > 0 [scalar]
        sampling rate of `y`

    onset_envelope     : np.ndarray [shape=(m,)]
        (optional) pre-computed onset strength envelope

    hop_length : int > 0 [scalar]
        hop length (in samples)

    units : {'frames', 'samples', 'time'}
        The units to encode detected onset events in.
        By default, 'frames' are used.

    backtrack : bool
        If `True`, detected onset events are backtracked to the nearest
        preceding minimum of `energy`.

        This is primarily useful when using onsets as slice points for segmentation.

    energy : np.ndarray [shape=(m,)] (optional)
        An energy function to use for backtracking detected onset events.
        If none is provided, then `onset_envelope` is used.

    kwargs : additional keyword arguments
        Additional parameters for peak picking.

        See `librosa.util.peak_pick` for details.

    Returns
    -------

    onsets : np.ndarray [shape=(n_onsets,)]
        estimated positions of detected onsets, in whichever units
        are specified.  By default, frame indices.

        .. note::
            If no onset strength could be detected, onset_detect returns
            an empty list.

    Raises
    ------
    ParameterError
        if neither `y` nor `onsets` are provided

        or if `units` is not one of 'frames', 'samples', or 'time'

    See Also
    --------
    onset_strength : compute onset strength per-frame
    onset_backtrack : backtracking onset events
    librosa.util.peak_pick : pick peaks from a time series

    Examples
    --------
    Get onset times from a signal

    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      offset=30, duration=2.0)
    >>> onset_frames = librosa.onset.onset_detect(y=y, sr=sr)
    >>> librosa.frames_to_time(onset_frames, sr=sr)
    array([ 0.07 ,  0.395,  0.511,  0.627,  0.766,  0.975,
            1.207,  1.324,  1.44 ,  1.788,  1.881])

    Or use a pre-computed onset envelope

    >>> o_env = librosa.onset.onset_strength(y, sr=sr)
    >>> times = librosa.frames_to_time(np.arange(len(o_env)), sr=sr)
    >>> onset_frames = librosa.onset.onset_detect(onset_envelope=o_env, sr=sr)

    >>> import matplotlib.pyplot as plt
    >>> D = np.abs(librosa.stft(y))
    >>> plt.figure()
    >>> ax1 = plt.subplot(2, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),
    ...                          x_axis='time', y_axis='log')
    >>> plt.title('Power spectrogram')
    >>> plt.subplot(2, 1, 2, sharex=ax1)
    >>> plt.plot(times, o_env, label='Onset strength')
    >>> plt.vlines(times[onset_frames], 0, o_env.max(), color='r', alpha=0.9,
    ...            linestyle='--', label='Onsets')
    >>> plt.axis('tight')
    >>> plt.legend(frameon=True, framealpha=0.75)


def onset_strength(y=None, sr=<NUM_LIT>, S=None, lag=<NUM_LIT:1>, max_size=<NUM_LIT:1>,<EOL>ref=None,<EOL>detrend=False, center=True,<EOL>feature=None, aggregate=None,<EOL>centering=None,<EOL>**kwargs):

if aggregate is False:<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>odf_all = onset_strength_multi(y=y,<EOL>sr=sr,<EOL>S=S,<EOL>lag=lag,<EOL>max_size=max_size,<EOL>ref=ref,<EOL>detrend=detrend,<EOL>center=center,<EOL>feature=feature,<EOL>aggregate=aggregate,<EOL>channels=None,<EOL>**kwargs)<EOL>return odf_all[<NUM_LIT:0>]<EOL>

Compute a spectral flux onset strength envelope.

    Onset strength at time `t` is determined by:

    `mean_f max(0, S[f, t] - ref[f, t - lag])`

    where `ref` is `S` after local max filtering along the frequency
    axis [1]_.

    By default, if a time series `y` is provided, S will be the
    log-power Mel spectrogram.

    .. [1] Böck, Sebastian, and Gerhard Widmer.
           "Maximum filter vibrato suppression for onset detection."
           16th International Conference on Digital Audio Effects,
           Maynooth, Ireland. 2013.

    Parameters
    ----------
    y        : np.ndarray [shape=(n,)]
        audio time-series

    sr       : number > 0 [scalar]
        sampling rate of `y`

    S        : np.ndarray [shape=(d, m)]
        pre-computed (log-power) spectrogram

    lag      : int > 0
        time lag for computing differences

    max_size : int > 0
        size (in frequency bins) of the local max filter.
        set to `1` to disable filtering.

    ref : None or np.ndarray [shape=(d, m)]
        An optional pre-computed reference spectrum, of the same shape as `S`.
        If not provided, it will be computed from `S`.
        If provided, it will override any local max filtering governed by `max_size`.

    detrend : bool [scalar]
        Filter the onset strength to remove the DC component

    center : bool [scalar]
        Shift the onset function by `n_fft / (2 * hop_length)` frames

    feature : function
        Function for computing time-series features, eg, scaled spectrograms.
        By default, uses `librosa.feature.melspectrogram` with `fmax=11025.0`

    aggregate : function
        Aggregation function to use when combining onsets
        at different frequency bins.

        Default: `np.mean`

    kwargs : additional keyword arguments
        Additional parameters to `feature()`, if `S` is not provided.

    Returns
    -------
    onset_envelope   : np.ndarray [shape=(m,)]
        vector containing the onset strength envelope

    Raises
    ------
    ParameterError
        if neither `(y, sr)` nor `S` are provided

        or if `lag` or `max_size` are not positive integers

    See Also
    --------
    onset_detect
    onset_strength_multi

    Examples
    --------
    First, load some audio and plot the spectrogram

    >>> import matplotlib.pyplot as plt
    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      duration=10.0)
    >>> D = np.abs(librosa.stft(y))
    >>> times = librosa.frames_to_time(np.arange(D.shape[1]))
    >>> plt.figure()
    >>> ax1 = plt.subplot(2, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),
    ...                          y_axis='log', x_axis='time')
    >>> plt.title('Power spectrogram')

    Construct a standard onset function

    >>> onset_env = librosa.onset.onset_strength(y=y, sr=sr)
    >>> plt.subplot(2, 1, 2, sharex=ax1)
    >>> plt.plot(times, 2 + onset_env / onset_env.max(), alpha=0.8,
    ...          label='Mean (mel)')

    Median aggregation, and custom mel options

    >>> onset_env = librosa.onset.onset_strength(y=y, sr=sr,
    ...                                          aggregate=np.median,
    ...                                          fmax=8000, n_mels=256)
    >>> plt.plot(times, 1 + onset_env / onset_env.max(), alpha=0.8,
    ...          label='Median (custom mel)')

    Constant-Q spectrogram instead of Mel

    >>> onset_env = librosa.onset.onset_strength(y=y, sr=sr,
    ...                                          feature=librosa.cqt)
    >>> plt.plot(times, onset_env / onset_env.max(), alpha=0.8,
    ...          label='Mean (CQT)')
    >>> plt.legend(frameon=True, framealpha=0.75)
    >>> plt.ylabel('Normalized strength')
    >>> plt.yticks([])
    >>> plt.axis('tight')
    >>> plt.tight_layout()


<EOL>minima = np.flatnonzero((energy[<NUM_LIT:1>:-<NUM_LIT:1>] <= energy[:-<NUM_LIT:2>]) &<EOL>(energy[<NUM_LIT:1>:-<NUM_LIT:1>] < energy[<NUM_LIT:2>:]))<EOL>minima = util.fix_frames(<NUM_LIT:1> + minima, x_min=<NUM_LIT:0>)<EOL>return minima[util.match_events(events, minima, right=False)]<EOL>

Backtrack detected onset events to the nearest preceding local
    minimum of an energy function.

    This function can be used to roll back the timing of detected onsets
    from a detected peak amplitude to the preceding minimum.

    This is most useful when using onsets to determine slice points for
    segmentation, as described by [1]_.

    .. [1] Jehan, Tristan.
           "Creating music by listening"
           Doctoral dissertation
           Massachusetts Institute of Technology, 2005.

    Parameters
    ----------
    events : np.ndarray, dtype=int
        List of onset event frame indices, as computed by `onset_detect`

    energy : np.ndarray, shape=(m,)
        An energy function

    Returns
    -------
    events_backtracked : np.ndarray, shape=events.shape
        The input events matched to nearest preceding minima of `energy`.

    Examples
    --------
    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      offset=30, duration=2.0)
    >>> oenv = librosa.onset.onset_strength(y=y, sr=sr)
    >>> # Detect events without backtracking
    >>> onset_raw = librosa.onset.onset_detect(onset_envelope=oenv,
    ...                                        backtrack=False)
    >>> # Backtrack the events using the onset envelope
    >>> onset_bt = librosa.onset.onset_backtrack(onset_raw, oenv)
    >>> # Backtrack the events using the RMS values
    >>> rms = librosa.feature.rms(S=np.abs(librosa.stft(y=y)))
    >>> onset_bt_rms = librosa.onset.onset_backtrack(onset_raw, rms[0])

    >>> # Plot the results
    >>> import matplotlib.pyplot as plt
    >>> plt.figure()
    >>> plt.subplot(2,1,1)
    >>> plt.plot(oenv, label='Onset strength')
    >>> plt.vlines(onset_raw, 0, oenv.max(), label='Raw onsets')
    >>> plt.vlines(onset_bt, 0, oenv.max(), label='Backtracked', color='r')
    >>> plt.legend(frameon=True, framealpha=0.75)
    >>> plt.subplot(2,1,2)
    >>> plt.plot(rms[0], label='RMS')
    >>> plt.vlines(onset_bt_rms, 0, rms.max(), label='Backtracked (RMS)', color='r')
    >>> plt.legend(frameon=True, framealpha=0.75)


@cache(level=<NUM_LIT:30>)<EOL>def onset_strength_multi(y=None, sr=<NUM_LIT>, S=None, lag=<NUM_LIT:1>, max_size=<NUM_LIT:1>,<EOL>ref=None, detrend=False, center=True, feature=None,<EOL>aggregate=None, channels=None, **kwargs):

if feature is None:<EOL><INDENT>feature = melspectrogram<EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>)<EOL><DEDENT>if aggregate is None:<EOL><INDENT>aggregate = np.mean<EOL><DEDENT>if lag < <NUM_LIT:1> or not isinstance(lag, int):<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>if max_size < <NUM_LIT:1> or not isinstance(max_size, int):<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>if S is None:<EOL><INDENT>S = np.abs(feature(y=y, sr=sr, **kwargs))<EOL>S = core.power_to_db(S)<EOL><DEDENT>n_fft = kwargs.get('<STR_LIT>', <NUM_LIT>)<EOL>hop_length = kwargs.get('<STR_LIT>', <NUM_LIT>)<EOL>S = np.atleast_2d(S)<EOL>if ref is None:<EOL><INDENT>if max_size == <NUM_LIT:1>:<EOL><INDENT>ref = S<EOL><DEDENT>else:<EOL><INDENT>ref = scipy.ndimage.maximum_filter1d(S, max_size, axis=<NUM_LIT:0>)<EOL><DEDENT><DEDENT>elif ref.shape != S.shape:<EOL><INDENT>raise ParameterError('<STR_LIT>'.format(ref.shape, S.shape))<EOL><DEDENT>onset_env = S[:, lag:] - ref[:, :-lag]<EOL>onset_env = np.maximum(<NUM_LIT:0.0>, onset_env)<EOL>pad = True<EOL>if channels is None:<EOL><INDENT>channels = [slice(None)]<EOL><DEDENT>else:<EOL><INDENT>pad = False<EOL><DEDENT>if aggregate:<EOL><INDENT>onset_env = util.sync(onset_env, channels,<EOL>aggregate=aggregate,<EOL>pad=pad, axis=<NUM_LIT:0>)<EOL><DEDENT>pad_width = lag<EOL>if center:<EOL><INDENT>pad_width += n_fft // (<NUM_LIT:2> * hop_length)<EOL><DEDENT>onset_env = np.pad(onset_env, ([<NUM_LIT:0>, <NUM_LIT:0>], [int(pad_width), <NUM_LIT:0>]),<EOL>mode='<STR_LIT>')<EOL>if detrend:<EOL><INDENT>onset_env = scipy.signal.lfilter([<NUM_LIT:1.0>, -<NUM_LIT:1.0>], [<NUM_LIT:1.0>, -<NUM_LIT>],<EOL>onset_env, axis=-<NUM_LIT:1>)<EOL><DEDENT>if center:<EOL><INDENT>onset_env = onset_env[:, :S.shape[<NUM_LIT:1>]]<EOL><DEDENT>return onset_env<EOL>

Compute a spectral flux onset strength envelope across multiple channels.

    Onset strength for channel `i` at time `t` is determined by:

    `mean_{f in channels[i]} max(0, S[f, t+1] - S[f, t])`

    Parameters
    ----------
    y        : np.ndarray [shape=(n,)]
        audio time-series

    sr       : number > 0 [scalar]
        sampling rate of `y`

    S        : np.ndarray [shape=(d, m)]
        pre-computed (log-power) spectrogram

    lag      : int > 0
        time lag for computing differences

    max_size : int > 0
        size (in frequency bins) of the local max filter.
        set to `1` to disable filtering.

    ref : None or np.ndarray [shape=(d, m)]
        An optional pre-computed reference spectrum, of the same shape as `S`.
        If not provided, it will be computed from `S`.
        If provided, it will override any local max filtering governed by `max_size`.

    detrend : bool [scalar]
        Filter the onset strength to remove the DC component

    center : bool [scalar]
        Shift the onset function by `n_fft / (2 * hop_length)` frames

    feature : function
        Function for computing time-series features, eg, scaled spectrograms.
        By default, uses `librosa.feature.melspectrogram` with `fmax=11025.0`

    aggregate : function or False
        Aggregation function to use when combining onsets
        at different frequency bins.

        If `False`, then no aggregation is performed.

        Default: `np.mean`

    channels : list or None
        Array of channel boundaries or slice objects.
        If `None`, then a single channel is generated to span all bands.

    kwargs : additional keyword arguments
        Additional parameters to `feature()`, if `S` is not provided.

    Returns
    -------
    onset_envelope   : np.ndarray [shape=(n_channels, m)]
        array containing the onset strength envelope for each specified channel

    Raises
    ------
    ParameterError
        if neither `(y, sr)` nor `S` are provided

    See Also
    --------
    onset_strength

    Notes
    -----
    This function caches at level 30.

    Examples
    --------
    First, load some audio and plot the spectrogram

    >>> import matplotlib.pyplot as plt
    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      duration=10.0)
    >>> D = np.abs(librosa.stft(y))
    >>> plt.figure()
    >>> plt.subplot(2, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),
    ...                          y_axis='log')
    >>> plt.title('Power spectrogram')

    Construct a standard onset function over four sub-bands

    >>> onset_subbands = librosa.onset.onset_strength_multi(y=y, sr=sr,
    ...                                                     channels=[0, 32, 64, 96, 128])
    >>> plt.subplot(2, 1, 2)
    >>> librosa.display.specshow(onset_subbands, x_axis='time')
    >>> plt.ylabel('Sub-bands')
    >>> plt.title('Sub-band onset strength')


def decompose(S, n_components=None, transformer=None, sort=False, fit=True, **kwargs):

if transformer is None:<EOL><INDENT>if fit is False:<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>transformer = sklearn.decomposition.NMF(n_components=n_components,<EOL>**kwargs)<EOL><DEDENT>if n_components is None:<EOL><INDENT>n_components = S.shape[<NUM_LIT:0>]<EOL><DEDENT>if fit:<EOL><INDENT>activations = transformer.fit_transform(S.T).T<EOL><DEDENT>else:<EOL><INDENT>activations = transformer.transform(S.T).T<EOL><DEDENT>components = transformer.components_.T<EOL>if sort:<EOL><INDENT>components, idx = util.axis_sort(components, index=True)<EOL>activations = activations[idx]<EOL><DEDENT>return components, activations<EOL>

Decompose a feature matrix.

    Given a spectrogram `S`, produce a decomposition into `components`
    and `activations` such that `S ~= components.dot(activations)`.

    By default, this is done with with non-negative matrix factorization (NMF),
    but any `sklearn.decomposition`-type object will work.

    Parameters
    ----------
    S : np.ndarray [shape=(n_features, n_samples), dtype=float]
        The input feature matrix (e.g., magnitude spectrogram)

    n_components : int > 0 [scalar] or None
        number of desired components

        if None, then `n_features` components are used

    transformer : None or object
        If None, use `sklearn.decomposition.NMF`

        Otherwise, any object with a similar interface to NMF should work.
        `transformer` must follow the scikit-learn convention, where
        input data is `(n_samples, n_features)`.

        `transformer.fit_transform()` will be run on `S.T` (not `S`),
        the return value of which is stored (transposed) as `activations`

        The components will be retrieved as `transformer.components_.T`

        `S ~= np.dot(activations, transformer.components_).T`

        or equivalently:
        `S ~= np.dot(transformer.components_.T, activations.T)`

    sort : bool
        If `True`, components are sorted by ascending peak frequency.

        .. note:: If used with `transformer`, sorting is applied to copies
            of the decomposition parameters, and not to `transformer`'s
            internal parameters.

    fit : bool
        If `True`, components are estimated from the input ``S``.

        If `False`, components are assumed to be pre-computed and stored
        in ``transformer``, and are not changed.

    kwargs : Additional keyword arguments to the default transformer
        `sklearn.decomposition.NMF`

    Returns
    -------
    components: np.ndarray [shape=(n_features, n_components)]
        matrix of components (basis elements).

    activations: np.ndarray [shape=(n_components, n_samples)]
        transformed matrix/activation matrix

    Raises
    ------
    ParameterError
        if `fit` is False and no `transformer` object is provided.

    See Also
    --------
    sklearn.decomposition : SciKit-Learn matrix decomposition modules

    Examples
    --------
    Decompose a magnitude spectrogram into 32 components with NMF

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> S = np.abs(librosa.stft(y))
    >>> comps, acts = librosa.decompose.decompose(S, n_components=8)
    >>> comps
    array([[  1.876e-01,   5.559e-02, ...,   1.687e-01,   4.907e-02],
           [  3.148e-01,   1.719e-01, ...,   2.314e-01,   9.493e-02],
           ...,
           [  1.561e-07,   8.564e-08, ...,   7.167e-08,   4.997e-08],
           [  1.531e-07,   7.880e-08, ...,   5.632e-08,   4.028e-08]])
    >>> acts
    array([[  4.197e-05,   8.512e-03, ...,   3.056e-05,   9.159e-06],
           [  9.568e-06,   1.718e-02, ...,   3.322e-05,   7.869e-06],
           ...,
           [  5.982e-05,   1.311e-02, ...,  -0.000e+00,   6.323e-06],
           [  3.782e-05,   7.056e-03, ...,   3.290e-05,  -0.000e+00]])

    Sort components by ascending peak frequency

    >>> comps, acts = librosa.decompose.decompose(S, n_components=16,
    ...                                           sort=True)

    Or with sparse dictionary learning

    >>> import sklearn.decomposition
    >>> T = sklearn.decomposition.MiniBatchDictionaryLearning(n_components=16)
    >>> scomps, sacts = librosa.decompose.decompose(S, transformer=T, sort=True)

    >>> import matplotlib.pyplot as plt
    >>> plt.figure(figsize=(10,8))
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(S,
    ...                                                  ref=np.max),
    ...                          y_axis='log', x_axis='time')
    >>> plt.title('Input spectrogram')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.subplot(3, 2, 3)
    >>> librosa.display.specshow(librosa.amplitude_to_db(comps,
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Components')
    >>> plt.subplot(3, 2, 4)
    >>> librosa.display.specshow(acts, x_axis='time')
    >>> plt.ylabel('Components')
    >>> plt.title('Activations')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 3)
    >>> S_approx = comps.dot(acts)
    >>> librosa.display.specshow(librosa.amplitude_to_db(S_approx,
    ...                                                  ref=np.max),
    ...                          y_axis='log', x_axis='time')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Reconstructed spectrogram')
    >>> plt.tight_layout()


@cache(level=<NUM_LIT:30>)<EOL>def hpss(S, kernel_size=<NUM_LIT>, power=<NUM_LIT>, mask=False, margin=<NUM_LIT:1.0>):

if np.iscomplexobj(S):<EOL><INDENT>S, phase = core.magphase(S)<EOL><DEDENT>else:<EOL><INDENT>phase = <NUM_LIT:1><EOL><DEDENT>if np.isscalar(kernel_size):<EOL><INDENT>win_harm = kernel_size<EOL>win_perc = kernel_size<EOL><DEDENT>else:<EOL><INDENT>win_harm = kernel_size[<NUM_LIT:0>]<EOL>win_perc = kernel_size[<NUM_LIT:1>]<EOL><DEDENT>if np.isscalar(margin):<EOL><INDENT>margin_harm = margin<EOL>margin_perc = margin<EOL><DEDENT>else:<EOL><INDENT>margin_harm = margin[<NUM_LIT:0>]<EOL>margin_perc = margin[<NUM_LIT:1>]<EOL><DEDENT>if margin_harm < <NUM_LIT:1> or margin_perc < <NUM_LIT:1>:<EOL><INDENT>raise ParameterError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>harm = np.empty_like(S)<EOL>harm[:] = median_filter(S, size=(<NUM_LIT:1>, win_harm), mode='<STR_LIT>')<EOL>perc = np.empty_like(S)<EOL>perc[:] = median_filter(S, size=(win_perc, <NUM_LIT:1>), mode='<STR_LIT>')<EOL>split_zeros = (margin_harm == <NUM_LIT:1> and margin_perc == <NUM_LIT:1>)<EOL>mask_harm = util.softmask(harm, perc * margin_harm,<EOL>power=power,<EOL>split_zeros=split_zeros)<EOL>mask_perc = util.softmask(perc, harm * margin_perc,<EOL>power=power,<EOL>split_zeros=split_zeros)<EOL>if mask:<EOL><INDENT>return mask_harm, mask_perc<EOL><DEDENT>return ((S * mask_harm) * phase, (S * mask_perc) * phase)<EOL>

Median-filtering harmonic percussive source separation (HPSS).

    If `margin = 1.0`, decomposes an input spectrogram `S = H + P`
    where `H` contains the harmonic components,
    and `P` contains the percussive components.

    If `margin > 1.0`, decomposes an input spectrogram `S = H + P + R`
    where `R` contains residual components not included in `H` or `P`.

    This implementation is based upon the algorithm described by [1]_ and [2]_.

    .. [1] Fitzgerald, Derry.
        "Harmonic/percussive separation using median filtering."
        13th International Conference on Digital Audio Effects (DAFX10),
        Graz, Austria, 2010.

    .. [2] Driedger, Müller, Disch.
        "Extending harmonic-percussive separation of audio."
        15th International Society for Music Information Retrieval Conference (ISMIR 2014),
        Taipei, Taiwan, 2014.

    Parameters
    ----------
    S : np.ndarray [shape=(d, n)]
        input spectrogram. May be real (magnitude) or complex.

    kernel_size : int or tuple (kernel_harmonic, kernel_percussive)
        kernel size(s) for the median filters.

        - If scalar, the same size is used for both harmonic and percussive.
        - If tuple, the first value specifies the width of the
          harmonic filter, and the second value specifies the width
          of the percussive filter.

    power : float > 0 [scalar]
        Exponent for the Wiener filter when constructing soft mask matrices.

    mask : bool
        Return the masking matrices instead of components.

        Masking matrices contain non-negative real values that
        can be used to measure the assignment of energy from `S`
        into harmonic or percussive components.

        Components can be recovered by multiplying `S * mask_H`
        or `S * mask_P`.

    margin : float or tuple (margin_harmonic, margin_percussive)
        margin size(s) for the masks (as described in [2]_)

        - If scalar, the same size is used for both harmonic and percussive.
        - If tuple, the first value specifies the margin of the
          harmonic mask, and the second value specifies the margin
          of the percussive mask.

    Returns
    -------
    harmonic : np.ndarray [shape=(d, n)]
        harmonic component (or mask)

    percussive : np.ndarray [shape=(d, n)]
        percussive component (or mask)

    See Also
    --------
    util.softmask

    Notes
    -----
    This function caches at level 30.

    Examples
    --------
    Separate into harmonic and percussive

    >>> y, sr = librosa.load(librosa.util.example_audio_file(), duration=15)
    >>> D = librosa.stft(y)
    >>> H, P = librosa.decompose.hpss(D)

    >>> import matplotlib.pyplot as plt
    >>> plt.figure()
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(np.abs(D),
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Full power spectrogram')
    >>> plt.subplot(3, 1, 2)
    >>> librosa.display.specshow(librosa.amplitude_to_db(np.abs(H),
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Harmonic power spectrogram')
    >>> plt.subplot(3, 1, 3)
    >>> librosa.display.specshow(librosa.amplitude_to_db(np.abs(P),
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Percussive power spectrogram')
    >>> plt.tight_layout()

    Or with a narrower horizontal filter

    >>> H, P = librosa.decompose.hpss(D, kernel_size=(13, 31))

    Just get harmonic/percussive masks, not the spectra

    >>> mask_H, mask_P = librosa.decompose.hpss(D, mask=True)
    >>> mask_H
    array([[  1.000e+00,   1.469e-01, ...,   2.648e-03,   2.164e-03],
           [  1.000e+00,   2.368e-01, ...,   9.413e-03,   7.703e-03],
           ...,
           [  8.869e-01,   5.673e-02, ...,   4.603e-02,   1.247e-05],
           [  7.068e-01,   2.194e-02, ...,   4.453e-02,   1.205e-05]], dtype=float32)
    >>> mask_P
    array([[  2.858e-05,   8.531e-01, ...,   9.974e-01,   9.978e-01],
           [  1.586e-05,   7.632e-01, ...,   9.906e-01,   9.923e-01],
           ...,
           [  1.131e-01,   9.433e-01, ...,   9.540e-01,   1.000e+00],
           [  2.932e-01,   9.781e-01, ...,   9.555e-01,   1.000e+00]], dtype=float32)

    Separate into harmonic/percussive/residual components by using a margin > 1.0

    >>> H, P = librosa.decompose.hpss(D, margin=3.0)
    >>> R = D - (H+P)
    >>> y_harm = librosa.core.istft(H)
    >>> y_perc = librosa.core.istft(P)
    >>> y_resi = librosa.core.istft(R)

    Get a more isolated percussive component by widening its margin

    >>> H, P = librosa.decompose.hpss(D, margin=(1.0,5.0))