PRODUCTS_FORM_PREFIX = "<STR_LIT>"<EOL>VOUCHERS_FORM_PREFIX = "<STR_LIT>"<EOL>v = _handle_voucher(request, VOUCHERS_FORM_PREFIX)<EOL>voucher_form, voucher_handled = v<EOL>category_id = int(category_id)  <EOL>category = inventory.Category.objects.get(pk=category_id)<EOL>with BatchController.batch(request.user):<EOL><INDENT>products = ProductController.available_products(<EOL>request.user,<EOL>category=category,<EOL>)<EOL>if not products:<EOL><INDENT>messages.warning(<EOL>request,<EOL>(<EOL>"<STR_LIT>" +<EOL>category.name<EOL>),<EOL>)<EOL>return redirect("<STR_LIT>")<EOL><DEDENT>p = _handle_products(request, category, products, PRODUCTS_FORM_PREFIX)<EOL>products_form, discounts, products_handled = p<EOL><DEDENT>if request.POST and not voucher_handled and not products_form.errors:<EOL><INDENT>if products_form.has_changed():<EOL><INDENT>messages.success(<EOL>request,<EOL>"<STR_LIT>",<EOL>)<EOL><DEDENT>return redirect(review)<EOL><DEDENT>data = {<EOL>"<STR_LIT>": category,<EOL>"<STR_LIT>": discounts,<EOL>"<STR_LIT>": products_form,<EOL>"<STR_LIT>": voucher_form,<EOL>}<EOL>return render(request, "<STR_LIT>", data)<EOL>

Form for selecting products from an individual product category.

    Arguments:
        category_id (castable to int): The id of the category to display.

    Returns:
        redirect or render:
            If the form has been sucessfully submitted, redirect to
            ``dashboard``. Otherwise, render
            ``registrasion/product_category.html`` with data::

                {
                    "category": category,         # An inventory.Category for
                                                  # category_id
                    "discounts": discounts,       # A list of
                                                  # DiscountAndQuantity
                    "form": products_form,        # A form for selecting
                                                  # products
                    "voucher_form": voucher_form, # A form for entering a
                                                  # voucher code
                }


cls._enter_batch_context(user)<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>cls._exit_batch_context(user)<EOL><DEDENT>

Marks the entry point for a batch for the given user.


types = list(ConditionController._controllers())<EOL>discounttypes = [<EOL>i for i in types if issubclass(i, conditions.DiscountBase)<EOL>]<EOL>product_clauses = conditions.DiscountForProduct.objects.all()<EOL>product_clauses = product_clauses.select_related(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>)<EOL>category_clauses = conditions.DiscountForCategory.objects.all()<EOL>category_clauses = category_clauses.select_related(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>)<EOL>all_subsets = []<EOL>for discounttype in discounttypes:<EOL><INDENT>discounts = discounttype.objects.all()<EOL>ctrl = ConditionController.for_type(discounttype)<EOL>discounts = ctrl.pre_filter(discounts, user)<EOL>all_subsets.append(discounts)<EOL><DEDENT>filtered_discounts = list(itertools.chain(*all_subsets))<EOL>from_filter = dict((i.id, i) for i in filtered_discounts)<EOL>clause_sets = (<EOL>product_clauses.filter(discount__in=filtered_discounts),<EOL>category_clauses.filter(discount__in=filtered_discounts),<EOL>)<EOL>clause_sets = (<EOL>cls._annotate_with_past_uses(i, user) for i in clause_sets<EOL>)<EOL>discount_clauses = set(itertools.chain(*clause_sets))<EOL>for clause in discount_clauses:<EOL><INDENT>clause.discount = from_filter[clause.discount.id]<EOL><DEDENT>return discount_clauses<EOL>

Returns:
    Sequence[DiscountForProduct | DiscountForCategory]: All clauses
    that passed the filter function.


@functools.wraps(func)<EOL>def inner(self, *a, **k):<EOL><INDENT>self._fail_if_cart_is_not_active()<EOL>with transaction.atomic():<EOL><INDENT>with BatchController.batch(self.cart.user):<EOL><INDENT>memoised = self.for_user(self.cart.user)<EOL>memoised._modified_by_batch = True<EOL>return func(self, *a, **k)<EOL><DEDENT><DEDENT><DEDENT>return inner<EOL>

Decorator that makes the wrapped function raise ValidationError
    if we're doing something that could modify the cart.

    It also wraps the execution of this function in a database transaction,
    and marks the boundaries of a cart operations batch.


time = timezone.now()<EOL>time_elapsed_since_updated = (time - self.cart.time_last_updated)<EOL>residual = self.cart.reservation_duration - time_elapsed_since_updated<EOL>reservations = [datetime.timedelta(<NUM_LIT:0>), residual]<EOL>if len(self.cart.vouchers.all()) >= <NUM_LIT:1>:<EOL><INDENT>reservations.append(inventory.Voucher.RESERVATION_DURATION)<EOL><DEDENT>items = commerce.ProductItem.objects.filter(cart=self.cart)<EOL>agg = items.aggregate(Max("<STR_LIT>"))<EOL>product_max = agg["<STR_LIT>"]<EOL>if product_max is not None:<EOL><INDENT>reservations.append(product_max)<EOL><DEDENT>self.cart.time_last_updated = time<EOL>self.cart.reservation_duration = max(reservations)<EOL>

Updates the cart's time last updated value, which is used to
        determine whether the cart has reserved the items and discounts it
        holds.


cart = self.cart<EOL>user = self.cart.user<EOL>errors = []<EOL>try:<EOL><INDENT>self._test_vouchers(self.cart.vouchers.all())<EOL><DEDENT>except ValidationError as ve:<EOL><INDENT>errors.append(ve)<EOL><DEDENT>items = commerce.ProductItem.objects.filter(cart=cart)<EOL>items = items.select_related("<STR_LIT>", "<STR_LIT>")<EOL>product_quantities = list((i.product, i.quantity) for i in items)<EOL>try:<EOL><INDENT>self._test_limits(product_quantities)<EOL><DEDENT>except ValidationError as ve:<EOL><INDENT>self._append_errors(errors, ve)<EOL><DEDENT>try:<EOL><INDENT>self._test_required_categories()<EOL><DEDENT>except ValidationError as ve:<EOL><INDENT>self._append_errors(errors, ve)<EOL><DEDENT>products = [i.product for i in items]<EOL>discounts_with_quantity = DiscountController.available_discounts(<EOL>user,<EOL>[],<EOL>products,<EOL>)<EOL>discounts = set(i.discount.id for i in discounts_with_quantity)<EOL>discount_items = commerce.DiscountItem.objects.filter(cart=cart)<EOL>for discount_item in discount_items:<EOL><INDENT>discount = discount_item.discount<EOL>if discount.id not in discounts:<EOL><INDENT>errors.append(<EOL>ValidationError("<STR_LIT>")<EOL>)<EOL><DEDENT><DEDENT>if errors:<EOL><INDENT>raise ValidationError(errors)<EOL><DEDENT>

Determines whether the status of the current cart is valid;
        this is normally called before generating or paying an invoice


self.validate_cart()<EOL>cart = self.cart<EOL>cart.refresh_from_db()<EOL>elapsed = (timezone.now() - cart.time_last_updated)<EOL>if cart.reservation_duration - elapsed > timedelta:<EOL><INDENT>return<EOL><DEDENT>cart.time_last_updated = timezone.now()<EOL>cart.reservation_duration = timedelta<EOL>cart.save()<EOL>

Extends the reservation on this cart by the given timedelta.
        This can only be done if the current state of the cart is valid (i.e
        all items and discounts in the cart are still available.)

        Arguments:
            timedelta (timedelta): The amount of time to extend the cart by.
                The resulting reservation_duration will be now() + timedelta,
                unless the requested extension is *LESS* than the current
                reservation deadline.


products = inventory.Product.objects.all()<EOL>cart_filter = (<EOL>Q(productitem__cart__user=user) &<EOL>Q(productitem__cart__status=commerce.Cart.STATUS_PAID)<EOL>)<EOL>quantity = When(<EOL>cart_filter,<EOL>then='<STR_LIT>'<EOL>)<EOL>quantity_or_zero = Case(<EOL>quantity,<EOL>default=Value(<NUM_LIT:0>),<EOL>)<EOL>remainder = Case(<EOL>When(limit_per_user=None, then=Value(<NUM_LIT>)),<EOL>default=F('<STR_LIT>') - Sum(quantity_or_zero),<EOL>)<EOL>products = products.annotate(remainder=remainder)<EOL>return dict((product.id, product.remainder) for product in products)<EOL>

Return:
    Mapping[int->int]: A dictionary that maps the product ID to the
    user's remainder for that product.


def user_quantity_remaining(self, user, filtered=True):

if filtered:<EOL><INDENT>if hasattr(self.condition, "<STR_LIT>"):<EOL><INDENT>return self.condition.remainder<EOL><DEDENT><DEDENT>qs = type(self.condition).objects.filter(pk=self.condition.id)<EOL>qs = self.pre_filter(qs, user)<EOL>if len(qs) > <NUM_LIT:0>:<EOL><INDENT>return qs[<NUM_LIT:0>].remainder<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>

returns 0 if the date range is violated, otherwise, it will return
        the quantity remaining under the stock limit.

        The filter for this condition must add an annotation called "remainder"
        in order for this to work.


@classmethod<EOL><INDENT>def pre_filter(self, queryset, user):<DEDENT>

<EOL>queryset = queryset.filter(<EOL>proposal_kind__proposalbase__presentation__cancelled=False<EOL>)<EOL>u = user<EOL>user_is_presenter = Q(<EOL>is_presenter=True,<EOL>proposal_kind__proposalbase__presentation__speaker__user=u,<EOL>)<EOL>user_is_copresenter = Q(<EOL>is_copresenter=True,<EOL>proposal_kind__proposalbase__presentation__additional_speakers__user=u,  <EOL>)<EOL>return queryset.filter(user_is_presenter | user_is_copresenter)<EOL>

Returns all of the items from queryset which are enabled by a user
        being a presenter or copresenter of a non-cancelled proposal.


in_user_carts = Q(<EOL>enabling_category__product__productitem__cart__user=user<EOL>)<EOL>released = commerce.Cart.STATUS_RELEASED<EOL>in_released_carts = Q(<EOL>enabling_category__product__productitem__cart__status=released<EOL>)<EOL>queryset = queryset.filter(in_user_carts)<EOL>queryset = queryset.exclude(in_released_carts)<EOL>return queryset<EOL>

Returns all of the items from queryset where the user has a
        product from a category invoking that item's condition in one of their
        carts.


if filtered:<EOL><INDENT>return True  <EOL><DEDENT>return self.passes_filter(user)<EOL>

Returns True if this flag condition is met, otherwise returns
        False. It determines if the condition is met by calling pre_filter
        with a queryset containing only self.condition.


if self.invoice.is_void:<EOL><INDENT>raise ValidationError("<STR_LIT>")<EOL><DEDENT>amount = self.invoice.total_payments()<EOL>if amount == <NUM_LIT:0>:<EOL><INDENT>self.void()<EOL>return<EOL><DEDENT>CreditNoteController.generate_from_invoice(self.invoice, amount)<EOL>self.update_status()<EOL>

Refunds the invoice by generating a CreditNote for the value of
        all of the payments against the cart.

        The invoice is marked as refunded, and the underlying cart is marked
        as released.


<EOL>invoices = commerce.Invoice.objects.filter(<EOL>user=invoice.user,<EOL>status=commerce.Invoice.STATUS_UNPAID,<EOL>)<EOL>if invoices.count() > <NUM_LIT:1>:<EOL><INDENT>return<EOL><DEDENT>notes = commerce.CreditNote.unclaimed().filter(<EOL>invoice__user=invoice.user<EOL>)<EOL>for note in notes:<EOL><INDENT>try:<EOL><INDENT>CreditNoteController(note).apply_to_invoice(invoice)<EOL><DEDENT>except ValidationError:<EOL><INDENT>break<EOL><DEDENT><DEDENT>invoice.refresh_from_db()<EOL>

Applies the user's credit notes to the given invoice on creation.


self.invoice.refresh_from_db()<EOL>if self.invoice.cart:<EOL><INDENT>self.invoice.cart.refresh_from_db()<EOL><DEDENT>

Refreshes the underlying invoice and cart objects.


@classmethod<EOL><INDENT>def email_on_invoice_change(cls, invoice, old_status, new_status):<DEDENT>

<EOL>silent_status = [<EOL>commerce.Invoice.STATUS_VOID,<EOL>commerce.Invoice.STATUS_UNPAID,<EOL>]<EOL>if old_status == new_status:<EOL><INDENT>return<EOL><DEDENT>if False and new_status in silent_status:<EOL><INDENT>pass<EOL><DEDENT>cls.email(invoice, "<STR_LIT>")<EOL>

Sends out all of the necessary notifications that the status of the
        invoice has changed to:

        - Invoice is now paid
        - Invoice is now refunded


@classmethod<EOL><INDENT>def email(cls, invoice, kind):<DEDENT>

context = {<EOL>"<STR_LIT>": invoice,<EOL>}<EOL>send_email([invoice.user.email], kind, context=context)<EOL>

Sends out an e-mail notifying the user about something to do
        with that invoice.


self.invoice.status = commerce.Invoice.STATUS_VOID<EOL>self.invoice.save()<EOL>

Marks the invoice as refunded, and updates the attached cart if
        necessary.


f = [<EOL>{<EOL>_ItemQuantityProductsForm.CHOICE_FIELD: product.id,<EOL>_ItemQuantityProductsForm.QUANTITY_FIELD: quantity,<EOL>}<EOL>for product, quantity in product_quantities<EOL>if quantity > <NUM_LIT:0><EOL>]<EOL>return f<EOL>

Prepares initial data for an instance of this form.
        product_quantities is a sequence of (product,quantity) tuples


form_type = staff_products_form_factory(user)<EOL>return forms.formset_factory(form_type)<EOL>

Creates a formset of StaffProductsForm for the given user.


return iter([])<EOL>

Yields a sequence of (product, quantity) tuples from the
        cleaned form data.


<EOL>cat = inventory.Category<EOL>RENDER_TYPES = {<EOL>cat.RENDER_TYPE_QUANTITY: _QuantityBoxProductsForm,<EOL>cat.RENDER_TYPE_RADIO: _RadioButtonProductsForm,<EOL>cat.RENDER_TYPE_ITEM_QUANTITY: _ItemQuantityProductsForm,<EOL>cat.RENDER_TYPE_CHECKBOX: _CheckboxProductsForm,<EOL>}<EOL>class ProductsForm(RENDER_TYPES[category.render_type]):<EOL><INDENT>pass<EOL><DEDENT>products = list(products)<EOL>products.sort(key=lambda prod: prod.order)<EOL>ProductsForm.set_fields(category, products)<EOL>if category.render_type == inventory.Category.RENDER_TYPE_ITEM_QUANTITY:<EOL><INDENT>ProductsForm = forms.formset_factory(<EOL>ProductsForm,<EOL>formset=_ItemQuantityProductsFormSet,<EOL>)<EOL><DEDENT>return ProductsForm<EOL>

Produces an appropriate _ProductsForm subclass for the given render
    type.


products = inventory.Product.objects.all()<EOL>products = ProductController.available_products(user, products=products)<EOL>product_ids = [product.id for product in products]<EOL>product_set = inventory.Product.objects.filter(id__in=product_ids)<EOL>class StaffProductsForm(forms.Form):<EOL><INDENT>'''<STR_LIT>'''<EOL>product = forms.ModelChoiceField(<EOL>widget=forms.Select,<EOL>queryset=product_set,<EOL>)<EOL>quantity = forms.IntegerField(<EOL>min_value=<NUM_LIT:0>,<EOL>)<EOL><DEDENT>return StaffProductsForm<EOL>

Creates a StaffProductsForm that restricts the available products to
    those that are available to a user.


products = set()<EOL>all_products = set()<EOL>for form in self:<EOL><INDENT>if form.empty_permitted and not form.cleaned_data:<EOL><INDENT>continue<EOL><DEDENT>for product, quantity in form.product_quantities():<EOL><INDENT>all_products.add(product)<EOL>if quantity == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>if product in products:<EOL><INDENT>form.add_error(<EOL>_ItemQuantityProductsForm.CHOICE_FIELD,<EOL>"<STR_LIT>",<EOL>)<EOL>form.add_error(<EOL>_ItemQuantityProductsForm.QUANTITY_FIELD,<EOL>"<STR_LIT>",<EOL>)<EOL><DEDENT>products.add(product)<EOL>yield product, quantity<EOL><DEDENT><DEDENT>for product in (all_products - products):<EOL><INDENT>yield product, <NUM_LIT:0><EOL><DEDENT>


return None<EOL>

Returns:
    The name of a field that stores the attendee's name. This is used
    to pre-fill the attendee's name from their Speaker profile, if they
    have one.


<EOL>slf = AttendeeProfileBase.objects.get_subclass(id=self.id)<EOL>if type(slf).invoice_recipient != type(self).invoice_recipient:<EOL><INDENT>return type(slf).invoice_recipient(slf)<EOL><DEDENT>return slf.attendee.user.username<EOL>

Returns:
    A representation of this attendee profile for the purpose
    of rendering to an invoice. This should include any information
    that you'd usually include on an invoice. Override in subclasses.


return self.price * self.quantity<EOL>


payments = PaymentBase.objects.filter(invoice=self)<EOL>total_paid = payments.aggregate(Sum("<STR_LIT>"))["<STR_LIT>"] or <NUM_LIT:0><EOL>return total_paid<EOL>

Returns the total amount paid towards this invoice.


return self.value - self.total_payments()<EOL>

Returns the total balance remaining towards this invoice.


return itertools.chain(self.products.all(), self.categories.all())<EOL>

Returns all of the items affected by this condition.


load()<EOL>tab = crontab.CronTab(user=True)<EOL>for task in registry:<EOL><INDENT>tab.new(task.command, KRONOS_BREADCRUMB).setall(task.schedule)<EOL><DEDENT>tab.write()<EOL>return len(registry)<EOL>


autodiscover_modules('<STR_LIT>')<EOL>if PROJECT_MODULE:<EOL><INDENT>if '<STR_LIT:.>' in PROJECT_MODULE.__name__:<EOL><INDENT>try:<EOL><INDENT>import_module('<STR_LIT>' % '<STR_LIT:.>'.join(<EOL>PROJECT_MODULE.__name__.split('<STR_LIT:.>')[<NUM_LIT:0>:-<NUM_LIT:1>]))<EOL><DEDENT>except ImportError as e:<EOL><INDENT>if '<STR_LIT>' not in str(e):<EOL><INDENT>print(e)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for cmd, app in get_commands().items():<EOL><INDENT>try:<EOL><INDENT>load_command_class(app, cmd)<EOL><DEDENT>except django.core.exceptions.ImproperlyConfigured:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>

Load ``cron`` modules for applications listed in ``INSTALLED_APPS``.


overlay_names = []<EOL>for blob in self._blobservice.list_blobs(<EOL>self.uuid,<EOL>prefix=self.overlays_key_prefix<EOL>):<EOL><INDENT>overlay_file = blob.name.rsplit('<STR_LIT:/>', <NUM_LIT:1>)[-<NUM_LIT:1>]<EOL>overlay_name, ext = overlay_file.split('<STR_LIT:.>')<EOL>overlay_names.append(overlay_name)<EOL><DEDENT>return overlay_names<EOL>


blob_generator = self._blobservice.list_blobs(<EOL>self.uuid,<EOL>include='<STR_LIT>'<EOL>)<EOL>for blob in blob_generator:<EOL><INDENT>if '<STR_LIT:type>' in blob.metadata:<EOL><INDENT>if blob.metadata['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>handle = blob.metadata['<STR_LIT>']<EOL>yield handle<EOL><DEDENT><DEDENT><DEDENT>


metadata = {}<EOL>identifier = generate_identifier(handle)<EOL>prefix = self.fragments_key_prefix + '<STR_LIT:{}>'.format(identifier)<EOL>blob_generator = self._blobservice.list_blobs(<EOL>self.uuid,<EOL>include='<STR_LIT>',<EOL>prefix=prefix<EOL>)<EOL>for blob in blob_generator:<EOL><INDENT>metadata_key = blob.name.split('<STR_LIT:.>')[-<NUM_LIT:2>]<EOL>value_as_string = self.get_text(blob.name)<EOL>value = json.loads(value_as_string)<EOL>metadata[metadata_key] = value<EOL><DEDENT>return metadata<EOL>

Return dictionary containing all metadata associated with handle.

        In other words all the metadata added using the ``add_item_metadata``
        method.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :returns: dictionary containing item metadata


def fake_extract_from_dir(filename, fileobj, method,<EOL>options=OPTIONS_MAP, keywords=TOWER_KEYWORDS,<EOL>comment_tags=COMMENT_TAGS):

for lineno, message, comments, context in extract(method, fileobj, keywords,<EOL>comment_tags, options):<EOL><INDENT>yield filename, lineno, message, comments<EOL><DEDENT>

We use Babel's exctract_from_dir() to pull out our gettext
    strings.  In the tests, I don't have a directory of files, I have StringIO
    objects.  So, we fake the original function with this one.


def ungettext(singular, plural, number, context=None):

singular_stripped = strip_whitespace(singular)<EOL>plural_stripped = strip_whitespace(plural)<EOL>if context:<EOL><INDENT>singular = add_context(context, singular_stripped)<EOL>plural = add_context(context, plural_stripped)<EOL><DEDENT>else:<EOL><INDENT>singular = singular_stripped<EOL>plural = plural_stripped<EOL><DEDENT>ret = django_nugettext(singular, plural, number)<EOL>if ret == singular:<EOL><INDENT>return singular_stripped<EOL><DEDENT>elif ret == plural:<EOL><INDENT>return plural_stripped<EOL><DEDENT>return ret<EOL>

Always return a stripped string, localized if possible


if isinstance(message, basestring):<EOL><INDENT>message = strip_whitespace(message)<EOL><DEDENT>elif isinstance(message, tuple):<EOL><INDENT>if len(message) == <NUM_LIT:2>:<EOL><INDENT>message = add_context(message[<NUM_LIT:1>], message[<NUM_LIT:0>])<EOL><DEDENT>elif len(message) == <NUM_LIT:3>:<EOL><INDENT>if all(isinstance(x, basestring) for x in message[:<NUM_LIT:2>]):<EOL><INDENT>singular, plural, num = message<EOL>message = (strip_whitespace(singular),<EOL>strip_whitespace(plural),<EOL>num)<EOL><DEDENT><DEDENT>elif len(message) == <NUM_LIT:4>:<EOL><INDENT>singular, plural, num, ctxt = message<EOL>message = (add_context(ctxt, strip_whitespace(singular)),<EOL>add_context(ctxt, strip_whitespace(plural)),<EOL>num)<EOL><DEDENT><DEDENT>return message<EOL>

We piggyback on jinja2's babel_extract() (really, Babel's extract_*
    functions) but they don't support some things we need so this function will
    tweak the message.  Specifically:

        1) We strip whitespace from the msgid.  Jinja2 will only strip
            whitespace from the ends of a string so linebreaks show up in
            your .po files still.

        2) Babel doesn't support context (msgctxt).  We hack that in ourselves
            here.


stripped = strip_whitespace(message)<EOL>message = add_context(context, stripped) if context else stripped<EOL>ret = django_ugettext(message)<EOL>return stripped if ret == message else ret<EOL>


class Translation(object):<EOL><INDENT>"""<STR_LIT>"""<EOL>ugettext = staticmethod(ugettext)<EOL>ungettext = staticmethod(ungettext)<EOL><DEDENT>import jingo<EOL>jingo.env.install_gettext_translations(Translation)<EOL>

Install our gettext and ngettext functions into Jinja2's environment.


result = {}<EOL>for (k, v) in self.client.get(HASH_FREQUENCY_TABLE,<EOL>*[(h,) for h in hashes]):<EOL><INDENT>if v is None:<EOL><INDENT>v = <NUM_LIT:0><EOL><DEDENT>result[k[<NUM_LIT:0>]] = v<EOL><DEDENT>return result<EOL>

Get document frequencies for a list of hashes.

        This will return all zeros unless the index was written with
        `hash_frequencies` set.  If :data:`DOCUMENT_HASH_KEY` is
        included in `hashes`, that value will be returned with the
        total number of documents indexed.  If you are looking for
        documents with that hash, pass
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param hashes: hashes to query
        :paramtype hashes: list of :class:`int`
        :return: map from hash to document frequency


(_, h) = self.make_hash_kw(tok)<EOL>return h<EOL>

Get a Murmur hash for a token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.


def verify_md5(md5_expected, data, other_errors=None):

<EOL>)<EOL>!= %r = received md5' \ d5_recv))<EOL>

return True if okay, raise Exception if not


def __call__(self, t_path, name_info, i_str):

logger.info('<STR_LIT>',<EOL>t_path, name_info, i_str)<EOL>try:<EOL><INDENT>more_name_info = get_name_info(t_path, i_str=i_str,<EOL>chunk_type=self.chunk_type)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.critical('<STR_LIT>', t_path, i_str, exc_info=True)<EOL>raise<EOL><DEDENT>self.name_info = dict(name_info, **more_name_info)<EOL>if self.name_info['<STR_LIT>'] == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>o_path = self.s3key_name<EOL>logger.info('<STR_LIT>',<EOL>self.__class__.__name__, o_path, i_str, t_path)<EOL>t_path2 = self.prepare_on_disk(t_path)<EOL>data_len = os.path.getsize(t_path2)<EOL>if data_len == <NUM_LIT:0>:<EOL><INDENT>logger.critical('<STR_LIT>')<EOL><DEDENT>logger.debug('<STR_LIT>', data_len, t_path2)<EOL>self.put_data(o_path, t_path2, self.name_info['<STR_LIT>'], data_len)<EOL>self.cleanup(t_path, t_path2)<EOL>logger.info('<STR_LIT>',<EOL>self.__class__.__name__, i_str, o_path)<EOL>return [o_path]<EOL>

Load chunk from t_path and put it into the right place in s3
using the output_name template from the config


return '<STR_LIT>'.join(['<STR_LIT>'.format(j, job_status(master, j))<EOL>for j in jobs])<EOL>

Get a quick summary of the statuses of all of 'jobs' as a
    (printable) string.


ext = '<STR_LIT>' if chunker['<STR_LIT>'].lower() == '<STR_LIT>' else '<STR_LIT>'<EOL>for k in dummies.keys():<EOL><INDENT>new_k = '<STR_LIT:.>'.join([k, ext, '<STR_LIT>'])<EOL>dummies[new_k] = dummies.pop(k)<EOL><DEDENT>

now that to_s3_chunks uses the file extension to detect the
    compression and type, this test has to make real filenames with
    proper extesions.


if n <= len(self._prefix):<EOL><INDENT>result = self._prefix[:n]<EOL>self._prefix = self._prefix[n:]<EOL>return result<EOL><DEDENT>n -= len(self._prefix)<EOL>result = self._prefix + self.f.read(n)<EOL>self._prefix = "<STR_LIT>"<EOL>return result<EOL>

Read (up to) 'n' bytes from the underlying file.  If any bytes
        have been pushed in with _unread() those are returned first.


def _make_content_item(node, mime_type=None, alternate_data=None):

raw = node.data<EOL>if getattr(node, '<STR_LIT>', None) == '<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>raw = zlib.decompress(node.data)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>if alternate_data is not None:<EOL><INDENT>try:<EOL><INDENT>raw = zlib.decompress(alternate_data)<EOL><DEDENT>except Exception:<EOL><INDENT>raise exc  <EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if mime_type is None:<EOL><INDENT>mime_type = node.mime_type<EOL><DEDENT>raw = raw.decode('<STR_LIT:utf8>').encode('<STR_LIT:utf8>')<EOL>return streamcorpus.ContentItem(raw=raw, media_type=mime_type)<EOL>

Create a ContentItem from a node in the spinn3r data tree.

    The ContentItem is created with raw data set to ``node.data``,
    decompressed if the node's encoding is 'zlib', and UTF-8
    normalized, with a MIME type from ``node.mime_type``.

    ``node``
      the actual node from the spinn3r protobuf data
    ``mime_type``
      string MIME type to use (defaults to ``node.mime_type``)
    ``alternate_data``
      alternate (compressed) data to use, if ``node.data`` is missing
      or can't be decompressed


o = cls()<EOL>o.ParseFromString(self._read_block())<EOL>return o<EOL>

Read some protobuf-encoded object stored in a single block
        out of the file.


html = uni(html)<EOL>parser = XpathTextCollector()<EOL>prev_end = <NUM_LIT:0><EOL>prev_progress = True<EOL>for start, end in char_offsets:<EOL><INDENT>if start == end:<EOL><INDENT>yield None<EOL>continue<EOL><DEDENT>if not prev_progress:<EOL><INDENT>for i in xrange(prev_end, start):<EOL><INDENT>parser.feed(html[i])<EOL>prev_end += <NUM_LIT:1><EOL>if parser.made_progress:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not parser.made_progress:<EOL><INDENT>yield None<EOL>continue<EOL><DEDENT><DEDENT>if prev_end < start:<EOL><INDENT>parser.feed(html[prev_end:start])<EOL>if not parser.made_progress:<EOL><INDENT>parser.feed(html[start:end])<EOL>prev_progress = parser.made_progress<EOL>prev_end = end<EOL>yield None<EOL>continue<EOL><DEDENT><DEDENT>xstart = parser.xpath_offset()<EOL>parser.feed(html[start:end])<EOL>xend = parser.xpath_offset()<EOL>prev_end = end<EOL>if not parser.made_progress:<EOL><INDENT>prev_progress = False<EOL>yield None<EOL><DEDENT>else:<EOL><INDENT>prev_progress = True<EOL>yield XpathRange(xstart[<NUM_LIT:0>], xstart[<NUM_LIT:1>], xend[<NUM_LIT:0>], xend[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>parser.feed(html[prev_end:])<EOL>parser.close()<EOL>

Converts HTML and a sequence of char offsets to xpath offsets.

    Returns a generator of :class:`streamcorpus.XpathRange` objects
    in correspondences with the sequence of ``char_offsets`` given.
    Namely, each ``XpathRange`` should address precisely the same text
    as that ``char_offsets`` (sans the HTML).

    Depending on how ``char_offsets`` was tokenized, it's possible that
    some tokens cannot have their xpaths generated reliably. In this
    case, a ``None`` value is yielded instead of a ``XpathRange``.

    ``char_offsets`` must be a sorted and non-overlapping sequence of
    character ranges. They do not have to be contiguous.


datai = self.depth_stack[-<NUM_LIT:1>].text_index()<EOL>xpath = (u'<STR_LIT:/>' +<EOL>u'<STR_LIT:/>'.join(dse.xpath_piece()<EOL>for dse in self.depth_stack[:-<NUM_LIT:1>]) +<EOL>(u'<STR_LIT>' % datai))<EOL>return (xpath, self.data_start)<EOL>

Returns a tuple of ``(xpath, character offset)``.

        The ``xpath`` returned *uniquely* identifies the end of the
        text node most recently inserted. The character offsets
        indicates where the text inside the node ends. (When the text
        node is empty, the offset returned is `0`.)


import argparse<EOL>import sys<EOL>parser = argparse.ArgumentParser()<EOL>parser.add_argument('<STR_LIT:path>')<EOL>args = parser.parse_args()<EOL>html = open(args.path).read()<EOL>html = html.decode('<STR_LIT:utf8>')<EOL>cursor = <NUM_LIT:0><EOL>for s in non_tag_chars_from_raw(html):<EOL><INDENT>for c in s:<EOL><INDENT>if c != '<STR_LIT:U+0020>' and c != html[cursor]:<EOL><INDENT>import pdb; pdb.set_trace()<EOL><DEDENT>sys.stdout.write(c.encode('<STR_LIT:utf8>'))<EOL>sys.stdout.flush()<EOL>cursor += <NUM_LIT:1><EOL><DEDENT><DEDENT>

manual test loop for make_clean_visible_from_raw


raise NotImplementedError('<STR_LIT>')<EOL>

Do an orderly shutdown of this stage.

        If the stage spawns a child process, kill it, and do any other
        required cleanup.  :meth:`streamcorpus_pipeline.Pipeline._cleanup`
        will call this method on every batch transform stage, regardless
        of whether this is the currently running stage or not.


pass<EOL>

Do an orderly shutdown of this stage.

        Incremental transforms are generally simple Python code and
        a complicated shutdown is not required.  The pipeline does not
        call this method.


lp_parser = LingPipeParser(self.config)<EOL>lp_parser.set(ner_dom)<EOL>sentences = list( lp_parser.sentences() )<EOL>return sentences, lp_parser.relations, lp_parser.attributes<EOL>

parse the sentences and tokens out of the XML


<EOL>self.sent_pos = <NUM_LIT:0><EOL>mention_id = <NUM_LIT:0><EOL>while len(sentence_dom.childNodes) > <NUM_LIT:0>:<EOL><INDENT>node = sentence_dom.childNodes.pop(<NUM_LIT:0>)<EOL>if node.nodeType == node.TEXT_NODE:<EOL><INDENT>for line in node.data.splitlines(True):<EOL><INDENT>self._input_string = line<EOL>for start, end in self.word_tokenizer.span_tokenize(line):<EOL><INDENT>tok = self._make_token(start, end)<EOL>if tok:<EOL><INDENT>yield tok<EOL><DEDENT><DEDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT>self.byte_idx += len(line.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert node.nodeName == '<STR_LIT>', node.nodeName<EOL>chain_id = node.attributes.get('<STR_LIT>').value<EOL>entity_type = node.attributes.get('<STR_LIT>').value<EOL>for node in node.childNodes:<EOL><INDENT>assert node.nodeType == node.TEXT_NODE, node.nodeType<EOL>for line in node.data.splitlines(True):<EOL><INDENT>self._input_string = line<EOL>for start, end in self.word_tokenizer.span_tokenize(line):<EOL><INDENT>tok = self._make_token(start, end)<EOL>if tok:<EOL><INDENT>if entity_type in _PRONOUNS:<EOL><INDENT>tok.mention_type = MentionType.PRO<EOL>tok.entity_type = _ENTITY_TYPES[entity_type]<EOL>attr = Attribute(<EOL>attribute_type=AttributeType.PER_GENDER,<EOL>value=str(_PRONOUNS[entity_type])<EOL>)<EOL>self.attributes.append(attr)<EOL><DEDENT>else:<EOL><INDENT>tok.mention_type = MentionType.NAME<EOL>tok.entity_type = _ENTITY_TYPES[entity_type]<EOL><DEDENT>tok.equiv_id = int(chain_id)<EOL>tok.mention_id = mention_id<EOL>yield tok<EOL><DEDENT><DEDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT>self.byte_idx += len(line.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT>mention_id += <NUM_LIT:1><EOL><DEDENT><DEDENT>

Tokenize all the words and preserve NER labels from ENAMEX tags


<EOL>tmp_chunk_path = chunk_path + '<STR_LIT:_>'<EOL>t_chunk = Chunk(path=tmp_chunk_path, mode='<STR_LIT:wb>')<EOL>for num, si in enumerate(Chunk(path=chunk_path)):<EOL><INDENT>if num < self.config['<STR_LIT>']:<EOL><INDENT>t_chunk.add(si)<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>t_chunk.close()<EOL>os.rename(tmp_chunk_path, chunk_path)<EOL>

batch-type transform stage: reads a chunk from chunk_path, and
replaces it with a new chunk at the same path


def align_chunk_with_ner(self, ner_xml_path, i_chunk, o_chunk):

<EOL>input_iter = i_chunk.__iter__()<EOL>all_ner = xml.dom.minidom.parse(open(ner_xml_path))<EOL>for ner_dom in all_ner.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>stream_item = next(input_iter)<EOL>stream_id = ner_dom.attributes.get('<STR_LIT>').value<EOL>if stream_item.stream_id is None:<EOL><INDENT>assert not stream_id, '<STR_LIT>' % stream_id<EOL>logger.critical('<STR_LIT>')<EOL>continue<EOL><DEDENT>assert stream_id and stream_id == stream_item.stream_id,'<STR_LIT>' % (stream_id, stream_item.stream_id)<EOL>if not stream_item.body:<EOL><INDENT>continue<EOL><DEDENT>tagging = Tagging()<EOL>tagging.tagger_id = self.tagger_id  <EOL>'''<STR_LIT>'''<EOL>tagging.generation_time = streamcorpus.make_stream_time()<EOL>stream_item.body.taggings[self.tagger_id] = tagging       <EOL>sentences, relations, attributes = self.get_sentences(ner_dom)<EOL>stream_item.body.sentences[self.tagger_id] = sentences    <EOL>stream_item.body.relations[self.tagger_id] = relations    <EOL>stream_item.body.attributes[self.tagger_id] = attributes  <EOL>logger.debug('<STR_LIT>' % stream_item.stream_id)<EOL>'''<STR_LIT>'''<EOL>if '<STR_LIT>' in self.config and self.config['<STR_LIT>']:<EOL><INDENT>assert '<STR_LIT>' in self.config, '<STR_LIT>'<EOL>aligner = AlignmentStrategies[ self.config['<STR_LIT>'] ]<EOL>aligner( stream_item, self.config['<STR_LIT>'] )<EOL><DEDENT>gc.collect()<EOL>try:<EOL><INDENT>o_chunk.add(stream_item)<EOL><DEDENT>except MemoryError as exc:<EOL><INDENT>msg = traceback.format_exc(exc)<EOL>msg += make_memory_info_msg()<EOL>logger.critical(msg)<EOL>raise PipelineOutOfMemory(msg)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>o_chunk.close()<EOL>logger.info('<STR_LIT>' % ner_xml_path)<EOL><DEDENT>except MemoryError as exc:<EOL><INDENT>msg = traceback.format_exc(exc)<EOL>msg += make_memory_info_msg()<EOL>logger.critical(msg)<EOL>raise PipelineOutOfMemory(msg)<EOL><DEDENT>

iterate through ner_xml_path to fuse with i_chunk into o_chunk


def make_ner_file(self, clean_visible_path, ner_xml_path):

if self.template is None:<EOL><INDENT>raise exceptions.NotImplementedError(

run tagger a child process to get XML output


chain_selector = aligner_data.get('<STR_LIT>', '<STR_LIT>')<EOL>assert chain_selector in _CHAIN_SELECTORS,'<STR_LIT>' % (chain_selector, list(_CHAIN_SELECTORS.keys()))<EOL>chain_selector = _CHAIN_SELECTORS[chain_selector]<EOL>equiv_ids = make_chains_with_names( stream_item.body.sentences )<EOL>required_annotator_id = aligner_data.get('<STR_LIT>')<EOL>for annotator_id, ratings in list(stream_item.ratings.items()):<EOL><INDENT>if (required_annotator_id is not None) and (annotator_id != required_annotator_id):<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>for rating in ratings:<EOL><INDENT>label = Label(annotator=rating.annotator,<EOL>target=rating.target)<EOL>for eqid, (chain_mentions, chain_tokens) in list(equiv_ids.items()):<EOL><INDENT>if chain_selector(rating.mentions, chain_mentions):<EOL><INDENT>for tok in chain_tokens:<EOL><INDENT>add_annotation(tok, label)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

Convert doc-level Rating object into a Label, and add that Label
to all Token in all coref chains identified by
aligner_data["chain_selector"]

:param stream_item: document that has a doc-level Rating to translate into token-level Labels.
:param aligner_data: dict containing:
  chain_selector: ALL or ANY
  annotator_id: string to find at stream_item.Ratings[i].annotator.annotator_id

If chain_selector==ALL, then only apply Label to chains in which
all of the Rating.mentions strings appear as substrings within at
least one of the Token.token strings.

If chain_selector==ANY, then apply Label to chains in which any of
the Rating.mentions strings appear as a substring within at least
one of the Token.token strings.

If chain_selector==ANY_MULTI_TOKEN, then apply Label to chains in which all
the names in any of the Rating.mentions strings appear as a substring within at least
one of the Token.token strings.


<EOL>fake_equiv_ids = -<NUM_LIT:2><EOL>equiv_ids = collections.defaultdict(lambda: (set(), set()))<EOL>for tagger_id, sents in list(sentences.items()):<EOL><INDENT>for sent in sents:<EOL><INDENT>for tok in sent.tokens:<EOL><INDENT>if tok.entity_type is not None:<EOL><INDENT>if tok.equiv_id == -<NUM_LIT:1>:<EOL><INDENT>eqid = fake_equiv_ids<EOL>fake_equiv_ids -= <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>eqid = tok.equiv_id<EOL><DEDENT>equiv_ids[eqid][<NUM_LIT:0>].add(cleanse(tok.token.decode('<STR_LIT:utf8>')))<EOL>equiv_ids[eqid][<NUM_LIT:1>].add(tok)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return equiv_ids<EOL>

assemble in-doc coref chains by mapping equiv_id to tokens and
their cleansed name strings

:param sentences: iterator over token generators
:returns dict:
    keys are equiv_ids,
    values are tuple(concatentated name string, list of tokens)


if isinstance(obj, (set, tuple, list)):<EOL><INDENT>return tuple([make_hash(e) for e in obj])<EOL><DEDENT>elif not isinstance(obj, dict):<EOL><INDENT>return hash(obj)<EOL><DEDENT>new_obj = copy.deepcopy(obj)<EOL>for k, v in new_obj.items():<EOL><INDENT>new_obj[k] = make_hash(v)<EOL><DEDENT>return hash(tuple(frozenset(new_obj.items())))<EOL>

Makes a hash from a dictionary, list, tuple or set to any level,
that contains only other hashable types (including any lists,
tuples, sets, and dictionaries).  See second answer (not the
accepted answer):
http://stackoverflow.com/questions/5884066/hashing-a-python-dictionary


previous_end = <NUM_LIT:0><EOL>clean_visible = clean_visible.decode('<STR_LIT:utf8>')<EOL>for start, end in self.sentence_tokenizer.span_tokenize(clean_visible):<EOL><INDENT>if start < previous_end:<EOL><INDENT>start = previous_end<EOL>if start > end:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>try:<EOL><INDENT>label = self.label_index.find_le(end)<EOL><DEDENT>except ValueError:<EOL><INDENT>label = None<EOL><DEDENT>if label:<EOL><INDENT>off = label.offsets[OffsetType.CHARS]<EOL>end = max(off.first + off.length, end)<EOL><DEDENT>previous_end = end<EOL>sent_str = clean_visible[start:end]<EOL>yield start, end, sent_str<EOL><DEDENT>

generate strings identified as sentences


def _process_output_chunk(self, start_count, next_idx, sources, i_str,<EOL>t_path):

if not self.t_chunk:<EOL><INDENT>return []<EOL><DEDENT>self.t_chunk.close()<EOL>o_paths = None<EOL>if len(self.t_chunk) > <NUM_LIT:0>:<EOL><INDENT>logger.info('<STR_LIT>',<EOL>len(self.t_chunk))<EOL>self._run_batch_transforms(t_path)<EOL>self._maybe_run_post_batch_incremental_transforms(t_path)<EOL>if (self.t_chunk) and (len(self.t_chunk) >= <NUM_LIT:0>):<EOL><INDENT>o_paths = self._run_writers(start_count, next_idx, sources,<EOL>i_str, t_path)<EOL><DEDENT><DEDENT>self.t_chunk = None<EOL>if self.work_unit and o_paths:<EOL><INDENT>old_o_paths = self.work_unit.data.get('<STR_LIT>', [])<EOL>o_paths = old_o_paths + o_paths<EOL>self.work_unit.data['<STR_LIT>'] = next_idx<EOL>self.work_unit.data['<STR_LIT>'] = o_paths<EOL>self.work_unit.update()<EOL><DEDENT>

for the current output chunk (which should be open):
  1. run batch transforms
  2. run post-batch incremental transforms
  3. run 'writers' to load-out the data to files or other storage
return list of paths that writers wrote to


tmp_dir_suffix = self.tmp_dir_suffix<EOL>if tmp_dir_suffix is None:<EOL><INDENT>with self.lock:<EOL><INDENT>self.tmp_dir_suffix = str(uuid.uuid4())<EOL>try:<EOL><INDENT>(reader, incremental_transforms, batch_transforms,<EOL>pbi_transforms, writers,<EOL>tmp_dir_path) = self._init_all_stages(config)<EOL><DEDENT>finally:<EOL><INDENT>self.tmp_dir_suffix = None<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>(reader, incremental_transforms, batch_transforms,<EOL>pbi_transforms, writers,<EOL>tmp_dir_path) = self._init_all_stages(config)<EOL><DEDENT>return Pipeline(<EOL>rate_log_interval=config['<STR_LIT>'],<EOL>input_item_limit=config.get('<STR_LIT>'),<EOL>cleanup_tmp_files=config['<STR_LIT>'],<EOL>tmp_dir_path=tmp_dir_path,<EOL>assert_single_source=config['<STR_LIT>'],<EOL>output_chunk_max_count=config.get('<STR_LIT>'),<EOL>output_max_clean_visible_bytes=config.get(<EOL>'<STR_LIT>'),<EOL>reader=reader,<EOL>incremental_transforms=incremental_transforms,<EOL>batch_transforms=batch_transforms,<EOL>post_batch_incremental_transforms=pbi_transforms,<EOL>writers=writers,<EOL>)<EOL>

Create a :class:`Pipeline`.

        Pass in the configuration under the ``streamcorpus_pipeline``
        block, not the top-level configuration that contains it.

        If :attr:`tmp_dir_suffix` is :const:`None`, then locks the
        factory and creates stages with a temporary (UUID) value.  If
        the configuration has `cleanup_tmp_files` set to :const:`True`
        (the default) then executing the resulting pipeline will clean
        up the directory afterwards.

        :param dict config: `streamcorpus_pipeline` configuration
        :return: new pipeline instance


return self.create(config[name], config)<EOL>

Create a single indirect stage.

        `name` should be the name of a config item that holds the
        name of a stage, for instance, ``reader``.  This looks up
        the name of that stage, then creates and returns the
        stage named.  For instance, if the config says

        .. code-block:: yaml

            reader: from_local_chunks

        then calling ``self._init_stage(scp_config, 'reader')`` will
        return a new instance of the
        :class:`~streamcorpus_pipeline._local_storage.from_local_chunks`
        stage.

        :param dict config: `streamcorpus_pipeline` configuration block
        :param str name: name of stage name entry
        :return: new instance of the stage


def create(self, stage, scp_config, config=None):

<EOL>if isinstance(stage, str):<EOL><INDENT>stage_name = stage<EOL>stage_obj = self.registry[stage_name]<EOL><DEDENT>else:<EOL><INDENT>stage_name = getattr(stage, '<STR_LIT>', stage.__name__)<EOL>stage_obj = stage<EOL><DEDENT>if config is None:<EOL><INDENT>config = scp_config.get(stage_name, None)<EOL><DEDENT>if config is None:<EOL><INDENT>config = getattr(stage_obj, '<STR_LIT>', {})<EOL><DEDENT>config = dict(config)<EOL>if self.tmp_dir_suffix is None:<EOL><INDENT>config['<STR_LIT>'] = scp_config['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>config['<STR_LIT>'] = os.path.join(scp_config['<STR_LIT>'],<EOL>self.tmp_dir_suffix)<EOL><DEDENT>config['<STR_LIT>'] = scp_config['<STR_LIT>']<EOL>return stage_obj(config)<EOL>

Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`


def __init__(self, rate_log_interval, input_item_limit,<EOL>cleanup_tmp_files, tmp_dir_path, assert_single_source,<EOL>output_chunk_max_count, output_max_clean_visible_bytes,<EOL>reader, incremental_transforms, batch_transforms,<EOL>post_batch_incremental_transforms, writers):

self.rate_log_interval = rate_log_interval<EOL>self.input_item_limit = input_item_limit<EOL>self.cleanup_tmp_files = cleanup_tmp_files<EOL>self.tmp_dir_path = tmp_dir_path<EOL>self.assert_single_source = assert_single_source<EOL>self.output_chunk_max_count = output_chunk_max_count<EOL>self.output_max_clean_visible_bytes = output_max_clean_visible_bytes<EOL>self.reader = reader<EOL>self.incremental_transforms = incremental_transforms<EOL>self.batch_transforms = batch_transforms<EOL>self.pbi_stages = post_batch_incremental_transforms<EOL>self.writers = writers<EOL>self.t_chunk = None<EOL>self.context = dict(<EOL>i_str=None,<EOL>data=None,<EOL>)<EOL>self.work_unit = None<EOL>

Create a new pipeline object.

        .. todo:: make this callable with just the lists of stages
                  and give sane (language-level) defaults for the rest

        :param int rate_log_interval: print progress every time this
          many input items have been processed
        :param int input_item_limit: stop after this many items
        :param bool cleanup_tmp_files: delete `tmp_dir_path` after
          execution if true
        :param str tmp_dir_path: path for intermediate files
        :param bool assert_single_source: require all items to have
          the same source value if true
        :param int output_chunk_max_count: restart output after
          writing this many items
        :param int output_max_clean_visible_bytes: restart output after
          writing this much content
        :param callable reader: reader stage object
        :param incremental_transforms: single-item transformation stages
        :paramtype incremental_transforms: list of callable
        :param batch_transforms: chunk-file transformation stages
        :paramtype batch_transforms: list of callable
        :param post_batch_incremental_transforms: single-item transformation
          stages
        :paramtype post_batch_incremental_transforms: list of callable
        :param writers: output stages
        :paramtype writers: list of callable


def rmtree(path, use_shutil=True, followlinks=False, retries=<NUM_LIT:10>):

if use_shutil and not followlinks:<EOL><INDENT>try:<EOL><INDENT>shutil.rmtree(path)<EOL>return<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.info('<STR_LIT>', path)<EOL>logger.debug('<STR_LIT>', traceback.format_exc(exc))<EOL><DEDENT><DEDENT>if not os.path.isdir(path):<EOL><INDENT>os.remove(path)<EOL>return<EOL><DEDENT>for root, dir_names, file_names in os.walk(path, topdown=False, followlinks=followlinks):<EOL><INDENT>for fname in file_names:<EOL><INDENT>fpath = os.path.join(root, fname)<EOL>tries = <NUM_LIT:0><EOL>while tries < retries:<EOL><INDENT>tries += <NUM_LIT:1><EOL>try:<EOL><INDENT>os.remove(fpath)<EOL>break<EOL><DEDENT>except Exception as exc:<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>if os.path.exists(fpath):<EOL><INDENT>logger.critical('<STR_LIT>', fpath)<EOL>logger.critical('<STR_LIT>', traceback.format_exc(exc))<EOL><DEDENT><DEDENT>for dname in dir_names:<EOL><INDENT>full_path = os.path.join(root, dname)<EOL>if os.path.islink(full_path):<EOL><INDENT>real_path = os.path.realpath(full_path)<EOL>os.remove(full_path)<EOL>full_path = real_path<EOL><DEDENT>os.rmdir(full_path)<EOL><DEDENT><DEDENT>if os.path.exists(path):<EOL><INDENT>os.rmdir(path)<EOL><DEDENT>

remove all files and directories below path, including path
    itself; works even when shutil.rmtree fails because of read-only
    files in NFS and Windows.  Follows symlinks.

    `use_shutil` defaults to True; useful for testing

    `followlinks` defaults to False; if set to True, shutil.rmtree is
    not used.


def parse_keys_and_ranges(i_str, keyfunc, rangefunc):

while i_str:<EOL><INDENT>m = _STREAM_ID_RE.match(i_str)<EOL>if m:<EOL><INDENT>for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):<EOL><INDENT>yield retval<EOL><DEDENT>i_str = i_str[m.end():]<EOL>while i_str and ((i_str[<NUM_LIT:0>] == '<STR_LIT:U+002C>') or (i_str[<NUM_LIT:0>] == '<STR_LIT:;>')):<EOL><INDENT>i_str = i_str[<NUM_LIT:1>:]<EOL><DEDENT>continue<EOL><DEDENT>if len(i_str) == SI_KEY_LENGTH:<EOL><INDENT>key = parse_si_key(i_str)<EOL>for retval in keyfunc(key):<EOL><INDENT>yield retval<EOL><DEDENT>return<EOL><DEDENT>keya = i_str[:SI_KEY_LENGTH]<EOL>splitc = i_str[SI_KEY_LENGTH]<EOL>if splitc == '<STR_LIT:<>':<EOL><INDENT>keyb = i_str[SI_KEY_LENGTH+<NUM_LIT:1>:SI_KEY_LENGTH+<NUM_LIT:1>+SI_KEY_LENGTH]<EOL>i_str = i_str[SI_KEY_LENGTH+<NUM_LIT:1>+SI_KEY_LENGTH:]<EOL>keya = parse_si_key(keya)<EOL>keyb = parse_si_key(keyb)<EOL>for retval in rangefunc(keya, keyb):<EOL><INDENT>yield retval<EOL><DEDENT><DEDENT>elif splitc == '<STR_LIT:;>':<EOL><INDENT>keya = parse_si_key(keya)<EOL>for retval in keyfunc(keya):<EOL><INDENT>yield retval<EOL><DEDENT>i_str = i_str[SI_KEY_LENGTH+<NUM_LIT:1>+<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>', splitc, i_str)<EOL>return<EOL><DEDENT><DEDENT>

Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.


if client is None:<EOL><INDENT>client = kvlayer.client()<EOL>client.setup_namespace(STREAM_ITEM_TABLE_DEFS,<EOL>STREAM_ITEM_VALUE_DEFS)<EOL><DEDENT>doc_id_range = make_doc_id_range(doc_id)<EOL>for k, v in client.scan(STREAM_ITEMS_TABLE, doc_id_range):<EOL><INDENT>if v is not None:<EOL><INDENT>errors, bytestr = streamcorpus.decrypt_and_uncompress(v)<EOL>yield streamcorpus.deserialize(bytestr)<EOL><DEDENT><DEDENT>

Retrieve :class:`streamcorpus.StreamItem`s from :mod:`kvlayer`.

    Namely, it returns an iterator over all documents with the given
    docid. The docid should be an md5 hash of the document's abs_url.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str doc_id: doc id of documents to retrieve
    :return: generator of :class:`streamcorpus.StreamItem`


def align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path):

o_chunk = Chunk()<EOL>input_iter = i_chunk.__iter__()<EOL>ner = '<STR_LIT>'<EOL>stream_id = None<EOL>all_ner = xml.dom.minidom.parse(open(tmp_ner_path))<EOL>for raw_ner in all_ner.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>stream_item = next(input_iter)<EOL>stream_id = raw_ner.attributes.get('<STR_LIT>').value<EOL>assert stream_id and stream_id == stream_item.stream_id,'<STR_LIT>' % (stream_id, stream_item.stream_id, ner)<EOL>tagger_id = '<STR_LIT>'<EOL>tagging = Tagging()<EOL>tagging.tagger_id = tagger_id<EOL>tagged_doc = list(lingpipe.files(raw_ner.toxml()))[<NUM_LIT:0>][<NUM_LIT:1>]<EOL>tagging.raw_tagging = tagged_doc<EOL>tagging.generation_time = streamcorpus.make_stream_time()<EOL>stream_item.body.taggings[tagger_id] = tagging<EOL>sentences = list(lingpipe.sentences(tagged_doc))<EOL>assert stream_item.ratings[<NUM_LIT:0>].mentions, stream_item.stream_id<EOL>john_smith_label = Label()<EOL>john_smith_label.annotator = stream_item.ratings[<NUM_LIT:0>].annotator<EOL>john_smith_label.target_id = stream_item.ratings[<NUM_LIT:0>].target_id<EOL>equiv_ids = collections.defaultdict(lambda: set())<EOL>for sent in sentences:<EOL><INDENT>for tok in sent.tokens:<EOL><INDENT>if tok.entity_type is not None:<EOL><INDENT>equiv_ids[tok.equiv_id].add(cleanse(tok.token))<EOL><DEDENT><DEDENT><DEDENT>johnsmiths = set()<EOL>for equiv_id, names in list(equiv_ids.items()):<EOL><INDENT>_names = cleanse('<STR_LIT:U+0020>'.join(names))<EOL>if '<STR_LIT>' in _names and '<STR_LIT>' in _names:<EOL><INDENT>johnsmiths.add(equiv_id)<EOL><DEDENT><DEDENT>print(len(johnsmiths))<EOL>for sent in sentences:<EOL><INDENT>for tok in sent.tokens:<EOL><INDENT>if tok.equiv_id in johnsmiths:<EOL><INDENT>tok.labels = [john_smith_label]                <EOL><DEDENT><DEDENT><DEDENT>stream_item.body.sentences[tagger_id] = sentences<EOL>o_chunk.add(stream_item)<EOL><DEDENT>open(tmp_done_path, '<STR_LIT:wb>').write(str(o_chunk))<EOL>print('<STR_LIT>' % tmp_done_path)<EOL>

iterate through the i_chunk and tmp_ner_path to generate a new
Chunk with body.ner


def make_ner_file(tagger_id, tmp_cleansed_path, tmp_ner_path, pipeline_root):

params = dict(INPUT_FILE=tmp_cleansed_path,<EOL>OUTPUT_FILE=tmp_ner_path,<EOL>PIPELINE_ROOT=pipeline_root)<EOL>pipeline_cmd = pipeline_cmd_templates[tagger_id] % params<EOL>print(pipeline_cmd)<EOL>print('<STR_LIT>' % tmp_ner_path)<EOL>start_time = time.time()<EOL>gpg_child = subprocess.Popen(<EOL>pipeline_cmd,<EOL>stderr=subprocess.PIPE, shell=True)<EOL>s_out, errors = gpg_child.communicate()<EOL>assert gpg_child.returncode == <NUM_LIT:0> and '<STR_LIT>' not in errors, errors<EOL>elapsed = time.time() - start_time<EOL>print('<STR_LIT>' % (tmp_ner_path, elapsed))<EOL>'''<STR_LIT>'''<EOL>


if self.matcher.match(strip_string(si.body.clean_visible.decode('<STR_LIT:utf-8>'))):<EOL><INDENT>return si<EOL><DEDENT>


options = _parse_args()<EOL>tarball = download_setuptools(download_base=options.download_base)<EOL>return _install(tarball, _build_install_args(options))<EOL>

Install or upgrade setuptools and EasyInstall


Called by the dashboard when the display should try and draw the notification.

Implementations should block the thread whilst the display is showing a notification, as soon as the thread is
unblocked then all data-sources will be polled.

:param notification: The notification to draw that is of a type returned by `get_supported_notifications()`


:return: Return an array of the types of outputs this notification produces. This is used to check the
display that's been configured can handle all the types of outputs the notification could produce.


return urllib.parse.quote(full_url, safe="<STR_LIT>")<EOL>

Encode invalid characters in URL to provide, such as spaces.

This implements code from the following URLs
https://bugs.python.org/issue14826
https://hg.python.org/cpython/rev/ebd37273e0fe


:param message: Message to filter
:return: True if the message should be kept, otherwise false.


Returns an array of all the component configs


self._text = text<EOL>self._source_name = str(source) if source else "<STR_LIT>"<EOL>

:param text: Entity's text
:param source: Source of the entity, which is converted to a string via `str()` to produce the source name


Argparse function to get the program parameters


print('<STR_LIT>', input_file)<EOL>y, sr = librosa.load(input_file)<EOL>print('<STR_LIT>')<EOL>y_harm = librosa.effects.harmonic(y)<EOL>print('<STR_LIT>')<EOL>tuning = librosa.estimate_tuning(y=y_harm, sr=sr)<EOL>print('<STR_LIT>'.format(<NUM_LIT:100> * tuning))<EOL>

Load an audio file and estimate tuning (in cents)


def onset_detect(y=None, sr=<NUM_LIT>, onset_envelope=None, hop_length=<NUM_LIT>,<EOL>backtrack=False, energy=None,<EOL>units='<STR_LIT>', **kwargs):

<EOL>if onset_envelope is None:<EOL><INDENT>if y is None:<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>onset_envelope = onset_strength(y=y, sr=sr, hop_length=hop_length)<EOL><DEDENT>onset_envelope -= onset_envelope.min()<EOL>if not onset_envelope.any():<EOL><INDENT>return np.array([], dtype=np.int)<EOL><DEDENT>onset_envelope /= onset_envelope.max()<EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length)       <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length + <NUM_LIT:1>)  <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length)       <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length + <NUM_LIT:1>)  <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>*sr//hop_length)          <EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>)<EOL>onsets = util.peak_pick(onset_envelope, **kwargs)<EOL>if backtrack:<EOL><INDENT>if energy is None:<EOL><INDENT>energy = onset_envelope<EOL><DEDENT>onsets = onset_backtrack(onsets, energy)<EOL><DEDENT>if units == '<STR_LIT>':<EOL><INDENT>pass<EOL><DEDENT>elif units == '<STR_LIT>':<EOL><INDENT>onsets = core.frames_to_samples(onsets, hop_length=hop_length)<EOL><DEDENT>elif units == '<STR_LIT:time>':<EOL><INDENT>onsets = core.frames_to_time(onsets, hop_length=hop_length, sr=sr)<EOL><DEDENT>else:<EOL><INDENT>raise ParameterError('<STR_LIT>'.format(units))<EOL><DEDENT>return onsets<EOL>

Basic onset detector.  Locate note onset events by picking peaks in an
    onset strength envelope.

    The `peak_pick` parameters were chosen by large-scale hyper-parameter
    optimization over the dataset provided by [1]_.

    .. [1] https://github.com/CPJKU/onset_db

    Parameters
    ----------
    y          : np.ndarray [shape=(n,)]
        audio time series

    sr         : number > 0 [scalar]
        sampling rate of `y`

    onset_envelope     : np.ndarray [shape=(m,)]
        (optional) pre-computed onset strength envelope

    hop_length : int > 0 [scalar]
        hop length (in samples)

    units : {'frames', 'samples', 'time'}
        The units to encode detected onset events in.
        By default, 'frames' are used.

    backtrack : bool
        If `True`, detected onset events are backtracked to the nearest
        preceding minimum of `energy`.

        This is primarily useful when using onsets as slice points for segmentation.

    energy : np.ndarray [shape=(m,)] (optional)
        An energy function to use for backtracking detected onset events.
        If none is provided, then `onset_envelope` is used.

    kwargs : additional keyword arguments
        Additional parameters for peak picking.

        See `librosa.util.peak_pick` for details.

    Returns
    -------

    onsets : np.ndarray [shape=(n_onsets,)]
        estimated positions of detected onsets, in whichever units
        are specified.  By default, frame indices.

        .. note::
            If no onset strength could be detected, onset_detect returns
            an empty list.

    Raises
    ------
    ParameterError
        if neither `y` nor `onsets` are provided

        or if `units` is not one of 'frames', 'samples', or 'time'

    See Also
    --------
    onset_strength : compute onset strength per-frame
    onset_backtrack : backtracking onset events
    librosa.util.peak_pick : pick peaks from a time series

    Examples
    --------
    Get onset times from a signal

    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      offset=30, duration=2.0)
    >>> onset_frames = librosa.onset.onset_detect(y=y, sr=sr)
    >>> librosa.frames_to_time(onset_frames, sr=sr)
    array([ 0.07 ,  0.395,  0.511,  0.627,  0.766,  0.975,
            1.207,  1.324,  1.44 ,  1.788,  1.881])

    Or use a pre-computed onset envelope

    >>> o_env = librosa.onset.onset_strength(y, sr=sr)
    >>> times = librosa.frames_to_time(np.arange(len(o_env)), sr=sr)
    >>> onset_frames = librosa.onset.onset_detect(onset_envelope=o_env, sr=sr)

    >>> import matplotlib.pyplot as plt
    >>> D = np.abs(librosa.stft(y))
    >>> plt.figure()
    >>> ax1 = plt.subplot(2, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),
    ...                          x_axis='time', y_axis='log')
    >>> plt.title('Power spectrogram')
    >>> plt.subplot(2, 1, 2, sharex=ax1)
    >>> plt.plot(times, o_env, label='Onset strength')
    >>> plt.vlines(times[onset_frames], 0, o_env.max(), color='r', alpha=0.9,
    ...            linestyle='--', label='Onsets')
    >>> plt.axis('tight')
    >>> plt.legend(frameon=True, framealpha=0.75)


@cache(level=<NUM_LIT:30>)<EOL>def onset_strength_multi(y=None, sr=<NUM_LIT>, S=None, lag=<NUM_LIT:1>, max_size=<NUM_LIT:1>,<EOL>ref=None, detrend=False, center=True, feature=None,<EOL>aggregate=None, channels=None, **kwargs):

if feature is None:<EOL><INDENT>feature = melspectrogram<EOL>kwargs.setdefault('<STR_LIT>', <NUM_LIT>)<EOL><DEDENT>if aggregate is None:<EOL><INDENT>aggregate = np.mean<EOL><DEDENT>if lag < <NUM_LIT:1> or not isinstance(lag, int):<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>if max_size < <NUM_LIT:1> or not isinstance(max_size, int):<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>if S is None:<EOL><INDENT>S = np.abs(feature(y=y, sr=sr, **kwargs))<EOL>S = core.power_to_db(S)<EOL><DEDENT>n_fft = kwargs.get('<STR_LIT>', <NUM_LIT>)<EOL>hop_length = kwargs.get('<STR_LIT>', <NUM_LIT>)<EOL>S = np.atleast_2d(S)<EOL>if ref is None:<EOL><INDENT>if max_size == <NUM_LIT:1>:<EOL><INDENT>ref = S<EOL><DEDENT>else:<EOL><INDENT>ref = scipy.ndimage.maximum_filter1d(S, max_size, axis=<NUM_LIT:0>)<EOL><DEDENT><DEDENT>elif ref.shape != S.shape:<EOL><INDENT>raise ParameterError('<STR_LIT>'.format(ref.shape, S.shape))<EOL><DEDENT>onset_env = S[:, lag:] - ref[:, :-lag]<EOL>onset_env = np.maximum(<NUM_LIT:0.0>, onset_env)<EOL>pad = True<EOL>if channels is None:<EOL><INDENT>channels = [slice(None)]<EOL><DEDENT>else:<EOL><INDENT>pad = False<EOL><DEDENT>if aggregate:<EOL><INDENT>onset_env = util.sync(onset_env, channels,<EOL>aggregate=aggregate,<EOL>pad=pad, axis=<NUM_LIT:0>)<EOL><DEDENT>pad_width = lag<EOL>if center:<EOL><INDENT>pad_width += n_fft // (<NUM_LIT:2> * hop_length)<EOL><DEDENT>onset_env = np.pad(onset_env, ([<NUM_LIT:0>, <NUM_LIT:0>], [int(pad_width), <NUM_LIT:0>]),<EOL>mode='<STR_LIT>')<EOL>if detrend:<EOL><INDENT>onset_env = scipy.signal.lfilter([<NUM_LIT:1.0>, -<NUM_LIT:1.0>], [<NUM_LIT:1.0>, -<NUM_LIT>],<EOL>onset_env, axis=-<NUM_LIT:1>)<EOL><DEDENT>if center:<EOL><INDENT>onset_env = onset_env[:, :S.shape[<NUM_LIT:1>]]<EOL><DEDENT>return onset_env<EOL>

Compute a spectral flux onset strength envelope across multiple channels.

    Onset strength for channel `i` at time `t` is determined by:

    `mean_{f in channels[i]} max(0, S[f, t+1] - S[f, t])`

    Parameters
    ----------
    y        : np.ndarray [shape=(n,)]
        audio time-series

    sr       : number > 0 [scalar]
        sampling rate of `y`

    S        : np.ndarray [shape=(d, m)]
        pre-computed (log-power) spectrogram

    lag      : int > 0
        time lag for computing differences

    max_size : int > 0
        size (in frequency bins) of the local max filter.
        set to `1` to disable filtering.

    ref : None or np.ndarray [shape=(d, m)]
        An optional pre-computed reference spectrum, of the same shape as `S`.
        If not provided, it will be computed from `S`.
        If provided, it will override any local max filtering governed by `max_size`.

    detrend : bool [scalar]
        Filter the onset strength to remove the DC component

    center : bool [scalar]
        Shift the onset function by `n_fft / (2 * hop_length)` frames

    feature : function
        Function for computing time-series features, eg, scaled spectrograms.
        By default, uses `librosa.feature.melspectrogram` with `fmax=11025.0`

    aggregate : function or False
        Aggregation function to use when combining onsets
        at different frequency bins.

        If `False`, then no aggregation is performed.

        Default: `np.mean`

    channels : list or None
        Array of channel boundaries or slice objects.
        If `None`, then a single channel is generated to span all bands.

    kwargs : additional keyword arguments
        Additional parameters to `feature()`, if `S` is not provided.

    Returns
    -------
    onset_envelope   : np.ndarray [shape=(n_channels, m)]
        array containing the onset strength envelope for each specified channel

    Raises
    ------
    ParameterError
        if neither `(y, sr)` nor `S` are provided

    See Also
    --------
    onset_strength

    Notes
    -----
    This function caches at level 30.

    Examples
    --------
    First, load some audio and plot the spectrogram

    >>> import matplotlib.pyplot as plt
    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      duration=10.0)
    >>> D = np.abs(librosa.stft(y))
    >>> plt.figure()
    >>> plt.subplot(2, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),
    ...                          y_axis='log')
    >>> plt.title('Power spectrogram')

    Construct a standard onset function over four sub-bands

    >>> onset_subbands = librosa.onset.onset_strength_multi(y=y, sr=sr,
    ...                                                     channels=[0, 32, 64, 96, 128])
    >>> plt.subplot(2, 1, 2)
    >>> librosa.display.specshow(onset_subbands, x_axis='time')
    >>> plt.ylabel('Sub-bands')
    >>> plt.title('Sub-band onset strength')


def onset_strength(y=None, sr=<NUM_LIT>, S=None, lag=<NUM_LIT:1>, max_size=<NUM_LIT:1>,<EOL>ref=None,<EOL>detrend=False, center=True,<EOL>feature=None, aggregate=None,<EOL>centering=None,<EOL>**kwargs):

if aggregate is False:<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>odf_all = onset_strength_multi(y=y,<EOL>sr=sr,<EOL>S=S,<EOL>lag=lag,<EOL>max_size=max_size,<EOL>ref=ref,<EOL>detrend=detrend,<EOL>center=center,<EOL>feature=feature,<EOL>aggregate=aggregate,<EOL>channels=None,<EOL>**kwargs)<EOL>return odf_all[<NUM_LIT:0>]<EOL>

Compute a spectral flux onset strength envelope.

    Onset strength at time `t` is determined by:

    `mean_f max(0, S[f, t] - ref[f, t - lag])`

    where `ref` is `S` after local max filtering along the frequency
    axis [1]_.

    By default, if a time series `y` is provided, S will be the
    log-power Mel spectrogram.

    .. [1] Böck, Sebastian, and Gerhard Widmer.
           "Maximum filter vibrato suppression for onset detection."
           16th International Conference on Digital Audio Effects,
           Maynooth, Ireland. 2013.

    Parameters
    ----------
    y        : np.ndarray [shape=(n,)]
        audio time-series

    sr       : number > 0 [scalar]
        sampling rate of `y`

    S        : np.ndarray [shape=(d, m)]
        pre-computed (log-power) spectrogram

    lag      : int > 0
        time lag for computing differences

    max_size : int > 0
        size (in frequency bins) of the local max filter.
        set to `1` to disable filtering.

    ref : None or np.ndarray [shape=(d, m)]
        An optional pre-computed reference spectrum, of the same shape as `S`.
        If not provided, it will be computed from `S`.
        If provided, it will override any local max filtering governed by `max_size`.

    detrend : bool [scalar]
        Filter the onset strength to remove the DC component

    center : bool [scalar]
        Shift the onset function by `n_fft / (2 * hop_length)` frames

    feature : function
        Function for computing time-series features, eg, scaled spectrograms.
        By default, uses `librosa.feature.melspectrogram` with `fmax=11025.0`

    aggregate : function
        Aggregation function to use when combining onsets
        at different frequency bins.

        Default: `np.mean`

    kwargs : additional keyword arguments
        Additional parameters to `feature()`, if `S` is not provided.

    Returns
    -------
    onset_envelope   : np.ndarray [shape=(m,)]
        vector containing the onset strength envelope

    Raises
    ------
    ParameterError
        if neither `(y, sr)` nor `S` are provided

        or if `lag` or `max_size` are not positive integers

    See Also
    --------
    onset_detect
    onset_strength_multi

    Examples
    --------
    First, load some audio and plot the spectrogram

    >>> import matplotlib.pyplot as plt
    >>> y, sr = librosa.load(librosa.util.example_audio_file(),
    ...                      duration=10.0)
    >>> D = np.abs(librosa.stft(y))
    >>> times = librosa.frames_to_time(np.arange(D.shape[1]))
    >>> plt.figure()
    >>> ax1 = plt.subplot(2, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),
    ...                          y_axis='log', x_axis='time')
    >>> plt.title('Power spectrogram')

    Construct a standard onset function

    >>> onset_env = librosa.onset.onset_strength(y=y, sr=sr)
    >>> plt.subplot(2, 1, 2, sharex=ax1)
    >>> plt.plot(times, 2 + onset_env / onset_env.max(), alpha=0.8,
    ...          label='Mean (mel)')

    Median aggregation, and custom mel options

    >>> onset_env = librosa.onset.onset_strength(y=y, sr=sr,
    ...                                          aggregate=np.median,
    ...                                          fmax=8000, n_mels=256)
    >>> plt.plot(times, 1 + onset_env / onset_env.max(), alpha=0.8,
    ...          label='Median (custom mel)')

    Constant-Q spectrogram instead of Mel

    >>> onset_env = librosa.onset.onset_strength(y=y, sr=sr,
    ...                                          feature=librosa.cqt)
    >>> plt.plot(times, onset_env / onset_env.max(), alpha=0.8,
    ...          label='Mean (CQT)')
    >>> plt.legend(frameon=True, framealpha=0.75)
    >>> plt.ylabel('Normalized strength')
    >>> plt.yticks([])
    >>> plt.axis('tight')
    >>> plt.tight_layout()


def __nn_filter_helper(R_data, R_indices, R_ptr, S, aggregate):

s_out = np.empty_like(S)<EOL>for i in range(len(R_ptr)-<NUM_LIT:1>):<EOL><INDENT>targets = R_indices[R_ptr[i]:R_ptr[i+<NUM_LIT:1>]]<EOL>if not len(targets):<EOL><INDENT>s_out[i] = S[i]<EOL>continue<EOL><DEDENT>neighbors = np.take(S, targets, axis=<NUM_LIT:0>)<EOL>if aggregate is np.average:<EOL><INDENT>weights = R_data[R_ptr[i]:R_ptr[i+<NUM_LIT:1>]]<EOL>s_out[i] = aggregate(neighbors, axis=<NUM_LIT:0>, weights=weights)<EOL><DEDENT>else:<EOL><INDENT>s_out[i] = aggregate(neighbors, axis=<NUM_LIT:0>)<EOL><DEDENT><DEDENT>return s_out<EOL>

Nearest-neighbor filter helper function.

    This is an internal function, not for use outside of the decompose module.

    It applies the nearest-neighbor filter to S, assuming that the first index
    corresponds to observations.

    Parameters
    ----------
    R_data, R_indices, R_ptr : np.ndarrays
        The `data`, `indices`, and `indptr` of a scipy.sparse matrix

    S : np.ndarray
        The observation data to filter

    aggregate : callable
        The aggregation operator

    Returns
    -------
    S_out : np.ndarray like S
        The filtered data array


@cache(level=<NUM_LIT:30>)<EOL>def hpss(S, kernel_size=<NUM_LIT>, power=<NUM_LIT>, mask=False, margin=<NUM_LIT:1.0>):

if np.iscomplexobj(S):<EOL><INDENT>S, phase = core.magphase(S)<EOL><DEDENT>else:<EOL><INDENT>phase = <NUM_LIT:1><EOL><DEDENT>if np.isscalar(kernel_size):<EOL><INDENT>win_harm = kernel_size<EOL>win_perc = kernel_size<EOL><DEDENT>else:<EOL><INDENT>win_harm = kernel_size[<NUM_LIT:0>]<EOL>win_perc = kernel_size[<NUM_LIT:1>]<EOL><DEDENT>if np.isscalar(margin):<EOL><INDENT>margin_harm = margin<EOL>margin_perc = margin<EOL><DEDENT>else:<EOL><INDENT>margin_harm = margin[<NUM_LIT:0>]<EOL>margin_perc = margin[<NUM_LIT:1>]<EOL><DEDENT>if margin_harm < <NUM_LIT:1> or margin_perc < <NUM_LIT:1>:<EOL><INDENT>raise ParameterError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>harm = np.empty_like(S)<EOL>harm[:] = median_filter(S, size=(<NUM_LIT:1>, win_harm), mode='<STR_LIT>')<EOL>perc = np.empty_like(S)<EOL>perc[:] = median_filter(S, size=(win_perc, <NUM_LIT:1>), mode='<STR_LIT>')<EOL>split_zeros = (margin_harm == <NUM_LIT:1> and margin_perc == <NUM_LIT:1>)<EOL>mask_harm = util.softmask(harm, perc * margin_harm,<EOL>power=power,<EOL>split_zeros=split_zeros)<EOL>mask_perc = util.softmask(perc, harm * margin_perc,<EOL>power=power,<EOL>split_zeros=split_zeros)<EOL>if mask:<EOL><INDENT>return mask_harm, mask_perc<EOL><DEDENT>return ((S * mask_harm) * phase, (S * mask_perc) * phase)<EOL>

Median-filtering harmonic percussive source separation (HPSS).

    If `margin = 1.0`, decomposes an input spectrogram `S = H + P`
    where `H` contains the harmonic components,
    and `P` contains the percussive components.

    If `margin > 1.0`, decomposes an input spectrogram `S = H + P + R`
    where `R` contains residual components not included in `H` or `P`.

    This implementation is based upon the algorithm described by [1]_ and [2]_.

    .. [1] Fitzgerald, Derry.
        "Harmonic/percussive separation using median filtering."
        13th International Conference on Digital Audio Effects (DAFX10),
        Graz, Austria, 2010.

    .. [2] Driedger, Müller, Disch.
        "Extending harmonic-percussive separation of audio."
        15th International Society for Music Information Retrieval Conference (ISMIR 2014),
        Taipei, Taiwan, 2014.

    Parameters
    ----------
    S : np.ndarray [shape=(d, n)]
        input spectrogram. May be real (magnitude) or complex.

    kernel_size : int or tuple (kernel_harmonic, kernel_percussive)
        kernel size(s) for the median filters.

        - If scalar, the same size is used for both harmonic and percussive.
        - If tuple, the first value specifies the width of the
          harmonic filter, and the second value specifies the width
          of the percussive filter.

    power : float > 0 [scalar]
        Exponent for the Wiener filter when constructing soft mask matrices.

    mask : bool
        Return the masking matrices instead of components.

        Masking matrices contain non-negative real values that
        can be used to measure the assignment of energy from `S`
        into harmonic or percussive components.

        Components can be recovered by multiplying `S * mask_H`
        or `S * mask_P`.

    margin : float or tuple (margin_harmonic, margin_percussive)
        margin size(s) for the masks (as described in [2]_)

        - If scalar, the same size is used for both harmonic and percussive.
        - If tuple, the first value specifies the margin of the
          harmonic mask, and the second value specifies the margin
          of the percussive mask.

    Returns
    -------
    harmonic : np.ndarray [shape=(d, n)]
        harmonic component (or mask)

    percussive : np.ndarray [shape=(d, n)]
        percussive component (or mask)

    See Also
    --------
    util.softmask

    Notes
    -----
    This function caches at level 30.

    Examples
    --------
    Separate into harmonic and percussive

    >>> y, sr = librosa.load(librosa.util.example_audio_file(), duration=15)
    >>> D = librosa.stft(y)
    >>> H, P = librosa.decompose.hpss(D)

    >>> import matplotlib.pyplot as plt
    >>> plt.figure()
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(np.abs(D),
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Full power spectrogram')
    >>> plt.subplot(3, 1, 2)
    >>> librosa.display.specshow(librosa.amplitude_to_db(np.abs(H),
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Harmonic power spectrogram')
    >>> plt.subplot(3, 1, 3)
    >>> librosa.display.specshow(librosa.amplitude_to_db(np.abs(P),
    ...                                                  ref=np.max),
    ...                          y_axis='log')
    >>> plt.colorbar(format='%+2.0f dB')
    >>> plt.title('Percussive power spectrogram')
    >>> plt.tight_layout()

    Or with a narrower horizontal filter

    >>> H, P = librosa.decompose.hpss(D, kernel_size=(13, 31))

    Just get harmonic/percussive masks, not the spectra

    >>> mask_H, mask_P = librosa.decompose.hpss(D, mask=True)
    >>> mask_H
    array([[  1.000e+00,   1.469e-01, ...,   2.648e-03,   2.164e-03],
           [  1.000e+00,   2.368e-01, ...,   9.413e-03,   7.703e-03],
           ...,
           [  8.869e-01,   5.673e-02, ...,   4.603e-02,   1.247e-05],
           [  7.068e-01,   2.194e-02, ...,   4.453e-02,   1.205e-05]], dtype=float32)
    >>> mask_P
    array([[  2.858e-05,   8.531e-01, ...,   9.974e-01,   9.978e-01],
           [  1.586e-05,   7.632e-01, ...,   9.906e-01,   9.923e-01],
           ...,
           [  1.131e-01,   9.433e-01, ...,   9.540e-01,   1.000e+00],
           [  2.932e-01,   9.781e-01, ...,   9.555e-01,   1.000e+00]], dtype=float32)

    Separate into harmonic/percussive/residual components by using a margin > 1.0

    >>> H, P = librosa.decompose.hpss(D, margin=3.0)
    >>> R = D - (H+P)
    >>> y_harm = librosa.core.istft(H)
    >>> y_perc = librosa.core.istft(P)
    >>> y_resi = librosa.core.istft(R)

    Get a more isolated percussive component by widening its margin

    >>> H, P = librosa.decompose.hpss(D, margin=(1.0,5.0))


def __trim_beats(localscore, beats, trim):

smooth_boe = scipy.signal.convolve(localscore[beats],<EOL>scipy.signal.hann(<NUM_LIT:5>),<EOL>'<STR_LIT>')<EOL>if trim:<EOL><INDENT>threshold = <NUM_LIT:0.5> * ((smooth_boe**<NUM_LIT:2>).mean()**<NUM_LIT:0.5>)<EOL><DEDENT>else:<EOL><INDENT>threshold = <NUM_LIT:0.0><EOL><DEDENT>valid = np.argwhere(smooth_boe > threshold)<EOL>return beats[valid.min():valid.max()]<EOL>

Final post-processing: throw out spurious leading/trailing beats


@cache(level=<NUM_LIT:30>)<EOL>def tempo(y=None, sr=<NUM_LIT>, onset_envelope=None, hop_length=<NUM_LIT>, start_bpm=<NUM_LIT>,<EOL>std_bpm=<NUM_LIT:1.0>, ac_size=<NUM_LIT>, max_tempo=<NUM_LIT>, aggregate=np.mean):

if start_bpm <= <NUM_LIT:0>:<EOL><INDENT>raise ParameterError('<STR_LIT>')<EOL><DEDENT>win_length = np.asscalar(core.time_to_frames(ac_size, sr=sr,<EOL>hop_length=hop_length))<EOL>tg = tempogram(y=y, sr=sr,<EOL>onset_envelope=onset_envelope,<EOL>hop_length=hop_length,<EOL>win_length=win_length)<EOL>if aggregate is not None:<EOL><INDENT>tg = aggregate(tg, axis=<NUM_LIT:1>, keepdims=True)<EOL><DEDENT>bpms = core.tempo_frequencies(tg.shape[<NUM_LIT:0>], hop_length=hop_length, sr=sr)<EOL>prior = np.exp(-<NUM_LIT:0.5> * ((np.log2(bpms) - np.log2(start_bpm)) / std_bpm)**<NUM_LIT:2>)<EOL>if max_tempo is not None:<EOL><INDENT>max_idx = np.argmax(bpms < max_tempo)<EOL>prior[:max_idx] = <NUM_LIT:0><EOL><DEDENT>best_period = np.argmax(tg * prior[:, np.newaxis], axis=<NUM_LIT:0>)<EOL>tempi = bpms[best_period]<EOL>tempi[best_period == <NUM_LIT:0>] = start_bpm<EOL>return tempi<EOL>

Estimate the tempo (beats per minute)

    Parameters
    ----------
    y : np.ndarray [shape=(n,)] or None
        audio time series

    sr : number > 0 [scalar]
        sampling rate of the time series

    onset_envelope    : np.ndarray [shape=(n,)]
        pre-computed onset strength envelope

    hop_length : int > 0 [scalar]
        hop length of the time series

    start_bpm : float [scalar]
        initial guess of the BPM

    std_bpm : float > 0 [scalar]
        standard deviation of tempo distribution

    ac_size : float > 0 [scalar]
        length (in seconds) of the auto-correlation window

    max_tempo : float > 0 [scalar, optional]
        If provided, only estimate tempo below this threshold

    aggregate : callable [optional]
        Aggregation function for estimating global tempo.
        If `None`, then tempo is estimated independently for each frame.

    Returns
    -------
    tempo : np.ndarray [scalar]
        estimated tempo (beats per minute)

    See Also
    --------
    librosa.onset.onset_strength
    librosa.feature.tempogram

    Notes
    -----
    This function caches at level 30.

    Examples
    --------
    >>> # Estimate a static tempo
    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> onset_env = librosa.onset.onset_strength(y, sr=sr)
    >>> tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sr)
    >>> tempo
    array([129.199])

    >>> # Or a dynamic tempo
    >>> dtempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sr,
    ...                             aggregate=None)
    >>> dtempo
    array([ 143.555,  143.555,  143.555, ...,  161.499,  161.499,
            172.266])

    Plot the estimated tempo against the onset autocorrelation

    >>> import matplotlib.pyplot as plt
    >>> # Convert to scalar
    >>> tempo = np.asscalar(tempo)
    >>> # Compute 2-second windowed autocorrelation
    >>> hop_length = 512
    >>> ac = librosa.autocorrelate(onset_env, 2 * sr // hop_length)
    >>> freqs = librosa.tempo_frequencies(len(ac), sr=sr,
    ...                                   hop_length=hop_length)
    >>> # Plot on a BPM axis.  We skip the first (0-lag) bin.
    >>> plt.figure(figsize=(8,4))
    >>> plt.semilogx(freqs[1:], librosa.util.normalize(ac)[1:],
    ...              label='Onset autocorrelation', basex=2)
    >>> plt.axvline(tempo, 0, 1, color='r', alpha=0.75, linestyle='--',
    ...            label='Tempo: {:.2f} BPM'.format(tempo))
    >>> plt.xlabel('Tempo (BPM)')
    >>> plt.grid()
    >>> plt.title('Static tempo estimation')
    >>> plt.legend(frameon=True)
    >>> plt.axis('tight')

    Plot dynamic tempo estimates over a tempogram

    >>> plt.figure()
    >>> tg = librosa.feature.tempogram(onset_envelope=onset_env, sr=sr,
    ...                                hop_length=hop_length)
    >>> librosa.display.specshow(tg, x_axis='time', y_axis='tempo')
    >>> plt.plot(librosa.frames_to_time(np.arange(len(dtempo))), dtempo,
    ...          color='w', linewidth=1.5, label='Tempo estimate')
    >>> plt.title('Dynamic tempo estimation')
    >>> plt.legend(frameon=True, framealpha=0.75)


if hasattr(window, '<STR_LIT>'):<EOL><INDENT>key = window.__name__<EOL><DEDENT>else:<EOL><INDENT>key = window<EOL><DEDENT>if key not in WINDOW_BANDWIDTHS:<EOL><INDENT>win = get_window(window, n)<EOL>WINDOW_BANDWIDTHS[key] = n * np.sum(win**<NUM_LIT:2>) / np.sum(np.abs(win))**<NUM_LIT:2><EOL><DEDENT>return WINDOW_BANDWIDTHS[key]<EOL>

Get the equivalent noise bandwidth of a window function.

    Parameters
    ----------
    window : callable or string
        A window function, or the name of a window function.
        Examples:
        - scipy.signal.hann
        - 'boxcar'

    n : int > 0
        The number of coefficients to use in estimating the
        window bandwidth

    Returns
    -------
    bandwidth : float
        The equivalent noise bandwidth (in FFT bins) of the
        given window function

    Notes
    -----
    This function caches at level 10.

    See Also
    --------
    get_window


@cache(level=<NUM_LIT:10>)<EOL>def mel(sr, n_fft, n_mels=<NUM_LIT>, fmin=<NUM_LIT:0.0>, fmax=None, htk=False,<EOL>norm=<NUM_LIT:1>, dtype=np.float32):

if fmax is None:<EOL><INDENT>fmax = float(sr) / <NUM_LIT:2><EOL><DEDENT>if norm is not None and norm != <NUM_LIT:1> and norm != np.inf:<EOL><INDENT>raise ParameterError('<STR_LIT>'.format(repr(norm)))<EOL><DEDENT>n_mels = int(n_mels)<EOL>weights = np.zeros((n_mels, int(<NUM_LIT:1> + n_fft // <NUM_LIT:2>)), dtype=dtype)<EOL>fftfreqs = fft_frequencies(sr=sr, n_fft=n_fft)<EOL>mel_f = mel_frequencies(n_mels + <NUM_LIT:2>, fmin=fmin, fmax=fmax, htk=htk)<EOL>fdiff = np.diff(mel_f)<EOL>ramps = np.subtract.outer(mel_f, fftfreqs)<EOL>for i in range(n_mels):<EOL><INDENT>lower = -ramps[i] / fdiff[i]<EOL>upper = ramps[i+<NUM_LIT:2>] / fdiff[i+<NUM_LIT:1>]<EOL>weights[i] = np.maximum(<NUM_LIT:0>, np.minimum(lower, upper))<EOL><DEDENT>if norm == <NUM_LIT:1>:<EOL><INDENT>enorm = <NUM_LIT> / (mel_f[<NUM_LIT:2>:n_mels+<NUM_LIT:2>] - mel_f[:n_mels])<EOL>weights *= enorm[:, np.newaxis]<EOL><DEDENT>if not np.all((mel_f[:-<NUM_LIT:2>] == <NUM_LIT:0>) | (weights.max(axis=<NUM_LIT:1>) > <NUM_LIT:0>)):<EOL><INDENT>warnings.warn('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>return weights<EOL>

Create a Filterbank matrix to combine FFT bins into Mel-frequency bins

    Parameters
    ----------
    sr        : number > 0 [scalar]
        sampling rate of the incoming signal

    n_fft     : int > 0 [scalar]
        number of FFT components

    n_mels    : int > 0 [scalar]
        number of Mel bands to generate

    fmin      : float >= 0 [scalar]
        lowest frequency (in Hz)

    fmax      : float >= 0 [scalar]
        highest frequency (in Hz).
        If `None`, use `fmax = sr / 2.0`

    htk       : bool [scalar]
        use HTK formula instead of Slaney

    norm : {None, 1, np.inf} [scalar]
        if 1, divide the triangular mel weights by the width of the mel band
        (area normalization).  Otherwise, leave all the triangles aiming for
        a peak value of 1.0

    dtype : np.dtype
        The data type of the output basis.
        By default, uses 32-bit (single-precision) floating point.

    Returns
    -------
    M         : np.ndarray [shape=(n_mels, 1 + n_fft/2)]
        Mel transform matrix

    Notes
    -----
    This function caches at level 10.

    Examples
    --------
    >>> melfb = librosa.filters.mel(22050, 2048)
    >>> melfb
    array([[ 0.   ,  0.016, ...,  0.   ,  0.   ],
           [ 0.   ,  0.   , ...,  0.   ,  0.   ],
           ...,
           [ 0.   ,  0.   , ...,  0.   ,  0.   ],
           [ 0.   ,  0.   , ...,  0.   ,  0.   ]])

    Clip the maximum frequency to 8KHz

    >>> librosa.filters.mel(22050, 2048, fmax=8000)
    array([[ 0.  ,  0.02, ...,  0.  ,  0.  ],
           [ 0.  ,  0.  , ...,  0.  ,  0.  ],
           ...,
           [ 0.  ,  0.  , ...,  0.  ,  0.  ],
           [ 0.  ,  0.  , ...,  0.  ,  0.  ]])

    >>> import matplotlib.pyplot as plt
    >>> plt.figure()
    >>> librosa.display.specshow(melfb, x_axis='linear')
    >>> plt.ylabel('Mel filter')
    >>> plt.title('Mel filter bank')
    >>> plt.colorbar()
    >>> plt.tight_layout()


def window_sumsquare(window, n_frames, hop_length=<NUM_LIT>, win_length=None, n_fft=<NUM_LIT>,<EOL>dtype=np.float32, norm=None):

if win_length is None:<EOL><INDENT>win_length = n_fft<EOL><DEDENT>n = n_fft + hop_length * (n_frames - <NUM_LIT:1>)<EOL>x = np.zeros(n, dtype=dtype)<EOL>win_sq = get_window(window, win_length)<EOL>win_sq = util.normalize(win_sq, norm=norm)**<NUM_LIT:2><EOL>win_sq = util.pad_center(win_sq, n_fft)<EOL>__window_ss_fill(x, win_sq, n_frames, hop_length)<EOL>return x<EOL>

Compute the sum-square envelope of a window function at a given hop length.

This is used to estimate modulation effects induced by windowing observations
in short-time fourier transforms.

Parameters
----------
window : string, tuple, number, callable, or list-like
    Window specification, as in `get_window`

n_frames : int > 0
    The number of analysis frames

hop_length : int > 0
    The number of samples to advance between frames

win_length : [optional]
    The length of the window function.  By default, this matches `n_fft`.

n_fft : int > 0
    The length of each analysis frame.

dtype : np.dtype
    The data type of the output

Returns
-------
wss : np.ndarray, shape=`(n_fft + hop_length * (n_frames - 1))`
    The sum-squared envelope of the window function

Examples
--------
For a fixed frame length (2048), compare modulation effects for a Hann window
at different hop lengths:

>>> n_frames = 50
>>> wss_256 = librosa.filters.window_sumsquare('hann', n_frames, hop_length=256)
>>> wss_512 = librosa.filters.window_sumsquare('hann', n_frames, hop_length=512)
>>> wss_1024 = librosa.filters.window_sumsquare('hann', n_frames, hop_length=1024)

>>> import matplotlib.pyplot as plt
>>> plt.figure()
>>> plt.subplot(3,1,1)
>>> plt.plot(wss_256)
>>> plt.title('hop_length=256')
>>> plt.subplot(3,1,2)
>>> plt.plot(wss_512)
>>> plt.title('hop_length=512')
>>> plt.subplot(3,1,3)
>>> plt.plot(wss_1024)
>>> plt.title('hop_length=1024')
>>> plt.tight_layout()


@jit(nopython=True, cache=True)<EOL>def __window_ss_fill(x, win_sq, n_frames, hop_length):

n = len(x)<EOL>n_fft = len(win_sq)<EOL>for i in range(n_frames):<EOL><INDENT>sample = i * hop_length<EOL>x[sample:min(n, sample + n_fft)] += win_sq[:max(<NUM_LIT:0>, min(n_fft, n - sample))]<EOL><DEDENT>

Helper function for window sum-square calculation.


@cache(level=<NUM_LIT:10>)<EOL>def get_window(window, Nx, fftbins=True):

if six.callable(window):<EOL><INDENT>return window(Nx)<EOL><DEDENT>elif (isinstance(window, (six.string_types, tuple)) or<EOL>np.isscalar(window)):<EOL><INDENT>return scipy.signal.get_window(window, Nx, fftbins=fftbins)<EOL><DEDENT>elif isinstance(window, (np.ndarray, list)):<EOL><INDENT>if len(window) == Nx:<EOL><INDENT>return np.asarray(window)<EOL><DEDENT>raise ParameterError('<STR_LIT>'<EOL>'<STR_LIT>'.format(len(window), Nx))<EOL><DEDENT>else:<EOL><INDENT>raise ParameterError('<STR_LIT>'.format(window))<EOL><DEDENT>

Compute a window function.

    This is a wrapper for `scipy.signal.get_window` that additionally
    supports callable or pre-computed windows.

    Parameters
    ----------
    window : string, tuple, number, callable, or list-like
        The window specification:

        - If string, it's the name of the window function (e.g., `'hann'`)
        - If tuple, it's the name of the window function and any parameters
          (e.g., `('kaiser', 4.0)`)
        - If numeric, it is treated as the beta parameter of the `'kaiser'`
          window, as in `scipy.signal.get_window`.
        - If callable, it's a function that accepts one integer argument
          (the window length)
        - If list-like, it's a pre-computed window of the correct length `Nx`

    Nx : int > 0
        The length of the window

    fftbins : bool, optional
        If True (default), create a periodic window for use with FFT
        If False, create a symmetric window for filter design applications.

    Returns
    -------
    get_window : np.ndarray
        A window of length `Nx` and type `window`

    See Also
    --------
    scipy.signal.get_window

    Notes
    -----
    This function caches at level 10.

    Raises
    ------
    ParameterError
        If `window` is supplied as a vector of length != `n_fft`,
        or is otherwise mis-specified.


@cache(level=<NUM_LIT:10>)<EOL>def constant_q(sr, fmin=None, n_bins=<NUM_LIT>, bins_per_octave=<NUM_LIT:12>, tuning=<NUM_LIT:0.0>,<EOL>window='<STR_LIT>', filter_scale=<NUM_LIT:1>, pad_fft=True, norm=<NUM_LIT:1>,<EOL>dtype=np.complex64, **kwargs):

if fmin is None:<EOL><INDENT>fmin = note_to_hz('<STR_LIT>')<EOL><DEDENT>lengths = constant_q_lengths(sr, fmin,<EOL>n_bins=n_bins,<EOL>bins_per_octave=bins_per_octave,<EOL>tuning=tuning,<EOL>window=window,<EOL>filter_scale=filter_scale)<EOL>correction = <NUM_LIT>**(float(tuning) / bins_per_octave)<EOL>fmin = correction * fmin<EOL>Q = float(filter_scale) / (<NUM_LIT>**(<NUM_LIT:1.> / bins_per_octave) - <NUM_LIT:1>)<EOL>freqs = Q * sr / lengths<EOL>filters = []<EOL>for ilen, freq in zip(lengths, freqs):<EOL><INDENT>sig = np.exp(np.arange(-ilen//<NUM_LIT:2>, ilen//<NUM_LIT:2>, dtype=float) * <NUM_LIT> * <NUM_LIT:2> * np.pi * freq / sr)<EOL>sig = sig * __float_window(window)(len(sig))<EOL>sig = util.normalize(sig, norm=norm)<EOL>filters.append(sig)<EOL><DEDENT>max_len = max(lengths)<EOL>if pad_fft:<EOL><INDENT>max_len = int(<NUM_LIT>**(np.ceil(np.log2(max_len))))<EOL><DEDENT>else:<EOL><INDENT>max_len = int(np.ceil(max_len))<EOL><DEDENT>filters = np.asarray([util.pad_center(filt, max_len, **kwargs)<EOL>for filt in filters], dtype=dtype)<EOL>return filters, np.asarray(lengths)<EOL>

r'''Construct a constant-Q basis.

    This uses the filter bank described by [1]_.

    .. [1] McVicar, Matthew.
            "A machine learning approach to automatic chord extraction."
            Dissertation, University of Bristol. 2013.

    Parameters
    ----------
    sr : number > 0 [scalar]
        Audio sampling rate

    fmin : float > 0 [scalar]
        Minimum frequency bin. Defaults to `C1 ~= 32.70`

    n_bins : int > 0 [scalar]
        Number of frequencies.  Defaults to 7 octaves (84 bins).

    bins_per_octave : int > 0 [scalar]
        Number of bins per octave

    tuning : float in `[-0.5, +0.5)` [scalar]
        Tuning deviation from A440 in fractions of a bin

    window : string, tuple, number, or function
        Windowing function to apply to filters.

    filter_scale : float > 0 [scalar]
        Scale of filter windows.
        Small values (<1) use shorter windows for higher temporal resolution.

    pad_fft : boolean
        Center-pad all filters up to the nearest integral power of 2.

        By default, padding is done with zeros, but this can be overridden
        by setting the `mode=` field in *kwargs*.

    norm : {inf, -inf, 0, float > 0}
        Type of norm to use for basis function normalization.
        See librosa.util.normalize

    dtype : np.dtype
        The data type of the output basis.
        By default, uses 64-bit (single precision) complex floating point.

    kwargs : additional keyword arguments
        Arguments to `np.pad()` when `pad==True`.

    Returns
    -------
    filters : np.ndarray, `len(filters) == n_bins`
        `filters[i]` is `i`\ th time-domain CQT basis filter

    lengths : np.ndarray, `len(lengths) == n_bins`
        The (fractional) length of each filter

    Notes
    -----
    This function caches at level 10.

    See Also
    --------
    constant_q_lengths
    librosa.core.cqt
    librosa.util.normalize

    Examples
    --------
    Use a shorter window for each filter

    >>> basis, lengths = librosa.filters.constant_q(22050, filter_scale=0.5)

    Plot one octave of filters in time and frequency

    >>> import matplotlib.pyplot as plt
    >>> basis, lengths = librosa.filters.constant_q(22050)
    >>> plt.figure(figsize=(10, 6))
    >>> plt.subplot(2, 1, 1)
    >>> notes = librosa.midi_to_note(np.arange(24, 24 + len(basis)))
    >>> for i, (f, n) in enumerate(zip(basis, notes[:12])):
    ...     f_scale = librosa.util.normalize(f) / 2
    ...     plt.plot(i + f_scale.real)
    ...     plt.plot(i + f_scale.imag, linestyle=':')
    >>> plt.axis('tight')
    >>> plt.yticks(np.arange(len(notes[:12])), notes[:12])
    >>> plt.ylabel('CQ filters')
    >>> plt.title('CQ filters (one octave, time domain)')
    >>> plt.xlabel('Time (samples at 22050 Hz)')
    >>> plt.legend(['Real', 'Imaginary'], frameon=True, framealpha=0.8)
    >>> plt.subplot(2, 1, 2)
    >>> F = np.abs(np.fft.fftn(basis, axes=[-1]))
    >>> # Keep only the positive frequencies
    >>> F = F[:, :(1 + F.shape[1] // 2)]
    >>> librosa.display.specshow(F, x_axis='linear')
    >>> plt.yticks(np.arange(len(notes))[::12], notes[::12])
    >>> plt.ylabel('CQ filters')
    >>> plt.title('CQ filter magnitudes (frequency domain)')
    >>> plt.tight_layout()


@cache(level=<NUM_LIT:10>)<EOL>def cq_to_chroma(n_input, bins_per_octave=<NUM_LIT:12>, n_chroma=<NUM_LIT:12>,<EOL>fmin=None, window=None, base_c=True, dtype=np.float32):

<EOL>n_merge = float(bins_per_octave) / n_chroma<EOL>if fmin is None:<EOL><INDENT>fmin = note_to_hz('<STR_LIT>')<EOL><DEDENT>if np.mod(n_merge, <NUM_LIT:1>) != <NUM_LIT:0>:<EOL><INDENT>raise ParameterError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>cq_to_ch = np.repeat(np.eye(n_chroma), n_merge, axis=<NUM_LIT:1>)<EOL>cq_to_ch = np.roll(cq_to_ch, - int(n_merge // <NUM_LIT:2>), axis=<NUM_LIT:1>)<EOL>n_octaves = np.ceil(np.float(n_input) / bins_per_octave)<EOL>cq_to_ch = np.tile(cq_to_ch, int(n_octaves))[:, :n_input]<EOL>midi_0 = np.mod(hz_to_midi(fmin), <NUM_LIT:12>)<EOL>if base_c:<EOL><INDENT>roll = midi_0<EOL><DEDENT>else:<EOL><INDENT>roll = midi_0 - <NUM_LIT:9><EOL><DEDENT>roll = int(np.round(roll * (n_chroma / <NUM_LIT>)))<EOL>cq_to_ch = np.roll(cq_to_ch, roll, axis=<NUM_LIT:0>).astype(dtype)<EOL>if window is not None:<EOL><INDENT>cq_to_ch = scipy.signal.convolve(cq_to_ch,<EOL>np.atleast_2d(window),<EOL>mode='<STR_LIT>')<EOL><DEDENT>return cq_to_ch<EOL>

Convert a Constant-Q basis to Chroma.

    Parameters
    ----------
    n_input : int > 0 [scalar]
        Number of input components (CQT bins)

    bins_per_octave : int > 0 [scalar]
        How many bins per octave in the CQT

    n_chroma : int > 0 [scalar]
        Number of output bins (per octave) in the chroma

    fmin : None or float > 0
        Center frequency of the first constant-Q channel.
        Default: 'C1' ~= 32.7 Hz

    window : None or np.ndarray
        If provided, the cq_to_chroma filter bank will be
        convolved with `window`.

    base_c : bool
        If True, the first chroma bin will start at 'C'
        If False, the first chroma bin will start at 'A'

    dtype : np.dtype
        The data type of the output basis.
        By default, uses 32-bit (single-precision) floating point.

    Returns
    -------
    cq_to_chroma : np.ndarray [shape=(n_chroma, n_input)]
        Transformation matrix: `Chroma = np.dot(cq_to_chroma, CQT)`

    Raises
    ------
    ParameterError
        If `n_input` is not an integer multiple of `n_chroma`

    Notes
    -----
    This function caches at level 10.

    Examples
    --------
    Get a CQT, and wrap bins to chroma

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> CQT = np.abs(librosa.cqt(y, sr=sr))
    >>> chroma_map = librosa.filters.cq_to_chroma(CQT.shape[0])
    >>> chromagram = chroma_map.dot(CQT)
    >>> # Max-normalize each time step
    >>> chromagram = librosa.util.normalize(chromagram, axis=0)

    >>> import matplotlib.pyplot as plt
    >>> plt.subplot(3, 1, 1)
    >>> librosa.display.specshow(librosa.amplitude_to_db(CQT,
    ...                                                  ref=np.max),
    ...                          y_axis='cqt_note')
    >>> plt.title('CQT Power')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 2)
    >>> librosa.display.specshow(chromagram, y_axis='chroma')
    >>> plt.title('Chroma (wrapped CQT)')
    >>> plt.colorbar()
    >>> plt.subplot(3, 1, 3)
    >>> chroma = librosa.feature.chroma_stft(y=y, sr=sr)
    >>> librosa.display.specshow(chroma, y_axis='chroma', x_axis='time')
    >>> plt.title('librosa.feature.chroma_stft')
    >>> plt.colorbar()
    >>> plt.tight_layout()


@jit(nopython=True, cache=True)<EOL>def __dtw_backtracking(D_steps, step_sizes_sigma):

wp = []<EOL>cur_idx = (D_steps.shape[<NUM_LIT:0>] - <NUM_LIT:1>, D_steps.shape[<NUM_LIT:1>] - <NUM_LIT:1>)<EOL>wp.append((cur_idx[<NUM_LIT:0>], cur_idx[<NUM_LIT:1>]))<EOL>while cur_idx[<NUM_LIT:0>] > <NUM_LIT:0>:<EOL><INDENT>cur_step_idx = D_steps[(cur_idx[<NUM_LIT:0>], cur_idx[<NUM_LIT:1>])]<EOL>cur_idx = (cur_idx[<NUM_LIT:0>] - step_sizes_sigma[cur_step_idx][<NUM_LIT:0>],<EOL>cur_idx[<NUM_LIT:1>] - step_sizes_sigma[cur_step_idx][<NUM_LIT:1>])<EOL>wp.append((cur_idx[<NUM_LIT:0>], cur_idx[<NUM_LIT:1>]))<EOL><DEDENT>return wp<EOL>

Backtrack optimal warping path.

    Uses the saved step sizes from the cost accumulation
    step to backtrack the index pairs for an optimal
    warping path.

    Parameters
    ----------
    D_steps : np.ndarray [shape=(N, M)]
        Saved indices of the used steps used in the calculation of D.

    step_sizes_sigma : np.ndarray [shape=[n, 2]]
        Specifies allowed step sizes as used by the dtw.

    Returns
    -------
    wp : list [shape=(N,)]
        Warping path with index pairs.
        Each list entry contains an index pair
        (n,m) as a tuple

    See Also
    --------
    dtw