self._refresh()<EOL>cart = self.invoice.cart<EOL>if not cart:<EOL><INDENT>return True<EOL><DEDENT>return cart.revision == self.invoice.cart_revision<EOL>

Returns true if there is no cart, or if the revision of this
        invoice matches the current revision of the cart.


is_valid = self._invoice_matches_cart()<EOL>cart = self.invoice.cart<EOL>if self.invoice.is_unpaid and is_valid and cart:<EOL><INDENT>try:<EOL><INDENT>CartController(cart).validate_cart()<EOL><DEDENT>except ValidationError:<EOL><INDENT>is_valid = False<EOL><DEDENT><DEDENT>if not is_valid:<EOL><INDENT>if self.invoice.total_payments() > <NUM_LIT:0>:<EOL><INDENT>self.refund()<EOL><DEDENT>else:<EOL><INDENT>self.void()<EOL><DEDENT><DEDENT>

Voids this invoice if the attached cart is no longer valid because
        the cart revision has changed, or the reservations have expired.


if self.invoice.total_payments() > <NUM_LIT:0>:<EOL><INDENT>raise ValidationError("<STR_LIT>")<EOL><DEDENT>elif self.invoice.is_refunded:<EOL><INDENT>raise ValidationError("<STR_LIT>")<EOL><DEDENT>if self.invoice.is_paid:<EOL><INDENT>self._release_cart()<EOL><DEDENT>self._mark_void()<EOL>

Voids the invoice if it is valid to do so.


if self.invoice.is_void:<EOL><INDENT>raise ValidationError("<STR_LIT>")<EOL><DEDENT>amount = self.invoice.total_payments()<EOL>if amount == <NUM_LIT:0>:<EOL><INDENT>self.void()<EOL>return<EOL><DEDENT>CreditNoteController.generate_from_invoice(self.invoice, amount)<EOL>self.update_status()<EOL>

Refunds the invoice by generating a CreditNote for the value of
        all of the payments against the cart.

        The invoice is marked as refunded, and the underlying cart is marked
        as released.


@classmethod<EOL><INDENT>def email(cls, invoice, kind):<DEDENT>

context = {<EOL>"<STR_LIT>": invoice,<EOL>}<EOL>send_email([invoice.user.email], kind, context=context)<EOL>

Sends out an e-mail notifying the user about something to do
        with that invoice.


cls.email(invoice, "<STR_LIT>")<EOL>

Sends out an e-mail notifying the user that an invoice has been
        created.


@classmethod<EOL><INDENT>def email_on_invoice_change(cls, invoice, old_status, new_status):<DEDENT>

<EOL>silent_status = [<EOL>commerce.Invoice.STATUS_VOID,<EOL>commerce.Invoice.STATUS_UNPAID,<EOL>]<EOL>if old_status == new_status:<EOL><INDENT>return<EOL><DEDENT>if False and new_status in silent_status:<EOL><INDENT>pass<EOL><DEDENT>cls.email(invoice, "<STR_LIT>")<EOL>

Sends out all of the necessary notifications that the status of the
        invoice has changed to:

        - Invoice is now paid
        - Invoice is now refunded


<EOL>cat = inventory.Category<EOL>RENDER_TYPES = {<EOL>cat.RENDER_TYPE_QUANTITY: _QuantityBoxProductsForm,<EOL>cat.RENDER_TYPE_RADIO: _RadioButtonProductsForm,<EOL>cat.RENDER_TYPE_ITEM_QUANTITY: _ItemQuantityProductsForm,<EOL>cat.RENDER_TYPE_CHECKBOX: _CheckboxProductsForm,<EOL>}<EOL>class ProductsForm(RENDER_TYPES[category.render_type]):<EOL><INDENT>pass<EOL><DEDENT>products = list(products)<EOL>products.sort(key=lambda prod: prod.order)<EOL>ProductsForm.set_fields(category, products)<EOL>if category.render_type == inventory.Category.RENDER_TYPE_ITEM_QUANTITY:<EOL><INDENT>ProductsForm = forms.formset_factory(<EOL>ProductsForm,<EOL>formset=_ItemQuantityProductsFormSet,<EOL>)<EOL><DEDENT>return ProductsForm<EOL>

Produces an appropriate _ProductsForm subclass for the given render
    type.


products = inventory.Product.objects.all()<EOL>products = ProductController.available_products(user, products=products)<EOL>product_ids = [product.id for product in products]<EOL>product_set = inventory.Product.objects.filter(id__in=product_ids)<EOL>class StaffProductsForm(forms.Form):<EOL><INDENT>'''<STR_LIT>'''<EOL>product = forms.ModelChoiceField(<EOL>widget=forms.Select,<EOL>queryset=product_set,<EOL>)<EOL>quantity = forms.IntegerField(<EOL>min_value=<NUM_LIT:0>,<EOL>)<EOL><DEDENT>return StaffProductsForm<EOL>

Creates a StaffProductsForm that restricts the available products to
    those that are available to a user.


form_type = staff_products_form_factory(user)<EOL>return forms.formset_factory(form_type)<EOL>

Creates a formset of StaffProductsForm for the given user.


self.user = user<EOL>super(ApplyCreditNoteForm, self).__init__(*a, **k)<EOL>self.fields["<STR_LIT>"].choices = self._unpaid_invoices<EOL>

User: The user whose invoices should be made available as
        choices.


@classmethod<EOL><INDENT>def set_fields(cls, category, products):<DEDENT>

pass<EOL>

Sets the base_fields on this _ProductsForm to allow selecting
        from the provided products.


return {}<EOL>

Prepares initial data for an instance of this form.
        product_quantities is a sequence of (product,quantity) tuples


return iter([])<EOL>

Yields a sequence of (product, quantity) tuples from the
        cleaned form data.


def add_product_error(self, product, error):

'''<STR_LIT>'''<EOL>self.add_error(self.field_name(product), error)<EOL>

Adds an error to the given product's field


f = [<EOL>{<EOL>_ItemQuantityProductsForm.CHOICE_FIELD: product.id,<EOL>_ItemQuantityProductsForm.QUANTITY_FIELD: quantity,<EOL>}<EOL>for product, quantity in product_quantities<EOL>if quantity > <NUM_LIT:0><EOL>]<EOL>return f<EOL>


products = set()<EOL>all_products = set()<EOL>for form in self:<EOL><INDENT>if form.empty_permitted and not form.cleaned_data:<EOL><INDENT>continue<EOL><DEDENT>for product, quantity in form.product_quantities():<EOL><INDENT>all_products.add(product)<EOL>if quantity == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>if product in products:<EOL><INDENT>form.add_error(<EOL>_ItemQuantityProductsForm.CHOICE_FIELD,<EOL>"<STR_LIT>",<EOL>)<EOL>form.add_error(<EOL>_ItemQuantityProductsForm.QUANTITY_FIELD,<EOL>"<STR_LIT>",<EOL>)<EOL><DEDENT>products.add(product)<EOL>yield product, quantity<EOL><DEDENT><DEDENT>for product in (all_products - products):<EOL><INDENT>yield product, <NUM_LIT:0><EOL><DEDENT>


try:<EOL><INDENT>return Attendee.objects.get(user=user)<EOL><DEDENT>except ObjectDoesNotExist:<EOL><INDENT>return Attendee.objects.create(user=user)<EOL><DEDENT>

Returns the instance of attendee for the given user, or creates
        a new one.


return None<EOL>

Returns:
    The name of a field that stores the attendee's name. This is used
    to pre-fill the attendee's name from their Speaker profile, if they
    have one.


<EOL>slf = AttendeeProfileBase.objects.get_subclass(id=self.id)<EOL>if type(slf).invoice_recipient != type(self).invoice_recipient:<EOL><INDENT>return type(slf).invoice_recipient(slf)<EOL><DEDENT>return slf.attendee.user.username<EOL>

Returns:
    A representation of this attendee profile for the purpose
    of rendering to an invoice. This should include any information
    that you'd usually include on an invoice. Override in subclasses.


self.code = self.normalise_code(self.code)<EOL>super(Voucher, self).save(*a, **k)<EOL>

Normalise the voucher code to be uppercase


return Cart.objects.filter(<EOL>(Q(status=Cart.STATUS_ACTIVE) &<EOL>Q(time_last_updated__gt=(<EOL>timezone.now()-F('<STR_LIT>')<EOL>))) |<EOL>Q(status=Cart.STATUS_PAID)<EOL>)<EOL>


payments = PaymentBase.objects.filter(invoice=self)<EOL>total_paid = payments.aggregate(Sum("<STR_LIT>"))["<STR_LIT>"] or <NUM_LIT:0><EOL>return total_paid<EOL>

Returns the total amount paid towards this invoice.


return self.value - self.total_payments()<EOL>

Returns the total balance remaining towards this invoice.


return self.price * self.quantity<EOL>


return -self.amount<EOL>

Returns the value of the credit note. Because CreditNotes are
        implemented as PaymentBase objects internally, the amount is a
        negative payment against an invoice.


products = self.discountforproduct_set.all()<EOL>categories = self.discountforcategory_set.all()<EOL>return itertools.chain(products, categories)<EOL>

Returns all of the effects of this discount.


return itertools.chain(self.products.all(), self.categories.all())<EOL>

Returns all of the items affected by this condition.


path = os.path.abspath(os.path.dirname(__file__))<EOL>filepath = os.path.join(path, filename)<EOL>try:<EOL><INDENT>return open(filepath).read()<EOL><DEDENT>except IOError:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>


autodiscover_modules('<STR_LIT>')<EOL>if PROJECT_MODULE:<EOL><INDENT>if '<STR_LIT:.>' in PROJECT_MODULE.__name__:<EOL><INDENT>try:<EOL><INDENT>import_module('<STR_LIT>' % '<STR_LIT:.>'.join(<EOL>PROJECT_MODULE.__name__.split('<STR_LIT:.>')[<NUM_LIT:0>:-<NUM_LIT:1>]))<EOL><DEDENT>except ImportError as e:<EOL><INDENT>if '<STR_LIT>' not in str(e):<EOL><INDENT>print(e)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for cmd, app in get_commands().items():<EOL><INDENT>try:<EOL><INDENT>load_command_class(app, cmd)<EOL><DEDENT>except django.core.exceptions.ImproperlyConfigured:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>

Load ``cron`` modules for applications listed in ``INSTALLED_APPS``.


load()<EOL>tab = crontab.CronTab(user=True)<EOL>for task in registry:<EOL><INDENT>tab.new(task.command, KRONOS_BREADCRUMB).setall(task.schedule)<EOL><DEDENT>tab.write()<EOL>return len(registry)<EOL>


load()<EOL>tab = crontab.CronTab('<STR_LIT>')<EOL>for task in registry:<EOL><INDENT>tab.new(task.command, KRONOS_BREADCRUMB).setall(task.schedule)<EOL><DEDENT>print(tab.render())<EOL>

Print the tasks that would be installed in the
crontab, for debugging purposes.


tab = crontab.CronTab(user=True)<EOL>count = len(list(tab.find_comment(KRONOS_BREADCRUMB)))<EOL>tab.remove_all(comment=KRONOS_BREADCRUMB)<EOL>tab.write()<EOL>return count<EOL>


@classmethod<EOL><INDENT>def list_dataset_uris(cls, base_uri, config_path):<DEDENT>

storage_account_name = generous_parse_uri(base_uri).netloc<EOL>blobservice = get_blob_service(storage_account_name, config_path)<EOL>containers = blobservice.list_containers(include_metadata=True)<EOL>uri_list = []<EOL>for c in containers:<EOL><INDENT>admin_metadata = c.metadata<EOL>uri = cls.generate_uri(<EOL>admin_metadata['<STR_LIT:name>'],<EOL>admin_metadata['<STR_LIT>'],<EOL>base_uri<EOL>)<EOL>uri_list.append(uri)<EOL><DEDENT>return uri_list<EOL>

Return list containing URIs with base URI.


try:<EOL><INDENT>self.get_admin_metadata()<EOL>return True<EOL><DEDENT>except (AzureMissingResourceHttpError, AzureHttpError):<EOL><INDENT>return False<EOL><DEDENT>

Return True if the administrative metadata exists.

        This is the definition of being a "dataset".


overlay_names = []<EOL>for blob in self._blobservice.list_blobs(<EOL>self.uuid,<EOL>prefix=self.overlays_key_prefix<EOL>):<EOL><INDENT>overlay_file = blob.name.rsplit('<STR_LIT:/>', <NUM_LIT:1>)[-<NUM_LIT:1>]<EOL>overlay_name, ext = overlay_file.split('<STR_LIT:.>')<EOL>overlay_names.append(overlay_name)<EOL><DEDENT>return overlay_names<EOL>


def add_item_metadata(self, handle, key, value):

identifier = generate_identifier(handle)<EOL>metadata_blob_suffix = "<STR_LIT>".format(identifier, key)<EOL>metadata_blob_name = self.fragments_key_prefix + metadata_blob_suffix<EOL>self._blobservice.create_blob_from_text(<EOL>self.uuid,<EOL>metadata_blob_name,<EOL>json.dumps(value)<EOL>)<EOL>self._blobservice.set_blob_metadata(<EOL>container_name=self.uuid,<EOL>blob_name=metadata_blob_name,<EOL>metadata={<EOL>"<STR_LIT:type>": "<STR_LIT>"<EOL>}<EOL>)<EOL>

Store the given key:value pair for the item associated with handle.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :param key: metadata key
        :param value: metadata value


self._blobservice.create_blob_from_text(<EOL>self.uuid,<EOL>key,<EOL>contents<EOL>)<EOL>

Store the given text contents so that they are later retrievable by
        the given key.


admin_metadata = self.get_admin_metadata()<EOL>uuid = admin_metadata["<STR_LIT>"]<EOL>dataset_cache_abspath = os.path.join(self._azure_cache_abspath, uuid)<EOL>mkdir_parents(dataset_cache_abspath)<EOL>metadata = self._blobservice.get_blob_metadata(<EOL>self.uuid,<EOL>identifier<EOL>)<EOL>relpath = metadata['<STR_LIT>']<EOL>_, ext = os.path.splitext(relpath)<EOL>local_item_abspath = os.path.join(<EOL>dataset_cache_abspath,<EOL>identifier + ext<EOL>)<EOL>if not os.path.isfile(local_item_abspath):<EOL><INDENT>tmp_local_item_abspath = local_item_abspath + "<STR_LIT>"<EOL>self._blobservice.get_blob_to_path(<EOL>self.uuid,<EOL>identifier,<EOL>tmp_local_item_abspath<EOL>)<EOL>os.rename(tmp_local_item_abspath, local_item_abspath)<EOL><DEDENT>return local_item_abspath<EOL>

Return absolute path at which item content can be accessed.

        :param identifier: item identifier
        :returns: absolute path from which the item content can be accessed


blob_generator = self._blobservice.list_blobs(<EOL>self.uuid,<EOL>include='<STR_LIT>'<EOL>)<EOL>for blob in blob_generator:<EOL><INDENT>if '<STR_LIT:type>' in blob.metadata:<EOL><INDENT>if blob.metadata['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>handle = blob.metadata['<STR_LIT>']<EOL>yield handle<EOL><DEDENT><DEDENT><DEDENT>


metadata = {}<EOL>identifier = generate_identifier(handle)<EOL>prefix = self.fragments_key_prefix + '<STR_LIT:{}>'.format(identifier)<EOL>blob_generator = self._blobservice.list_blobs(<EOL>self.uuid,<EOL>include='<STR_LIT>',<EOL>prefix=prefix<EOL>)<EOL>for blob in blob_generator:<EOL><INDENT>metadata_key = blob.name.split('<STR_LIT:.>')[-<NUM_LIT:2>]<EOL>value_as_string = self.get_text(blob.name)<EOL>value = json.loads(value_as_string)<EOL>metadata[metadata_key] = value<EOL><DEDENT>return metadata<EOL>

Return dictionary containing all metadata associated with handle.

        In other words all the metadata added using the ``add_item_metadata``
        method.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :returns: dictionary containing item metadata


account_key = get_config_value(<EOL>"<STR_LIT>" + account_name,<EOL>config_path=config_path<EOL>)<EOL>return account_key<EOL>

Return the Azure account key associated with the account name.


return binascii.hexlify(base64.b64decode(input_string)).decode()<EOL>

Return the hex encoded version of the base64 encoded input string.


def fake_extract_from_dir(filename, fileobj, method,<EOL>options=OPTIONS_MAP, keywords=TOWER_KEYWORDS,<EOL>comment_tags=COMMENT_TAGS):

for lineno, message, comments, context in extract(method, fileobj, keywords,<EOL>comment_tags, options):<EOL><INDENT>yield filename, lineno, message, comments<EOL><DEDENT>

We use Babel's exctract_from_dir() to pull out our gettext
    strings.  In the tests, I don't have a directory of files, I have StringIO
    objects.  So, we fake the original function with this one.


stripped = strip_whitespace(message)<EOL>message = add_context(context, stripped) if context else stripped<EOL>ret = django_ugettext(message)<EOL>return stripped if ret == message else ret<EOL>

Always return a stripped string, localized if possible


def ungettext(singular, plural, number, context=None):

singular_stripped = strip_whitespace(singular)<EOL>plural_stripped = strip_whitespace(plural)<EOL>if context:<EOL><INDENT>singular = add_context(context, singular_stripped)<EOL>plural = add_context(context, plural_stripped)<EOL><DEDENT>else:<EOL><INDENT>singular = singular_stripped<EOL>plural = plural_stripped<EOL><DEDENT>ret = django_nugettext(singular, plural, number)<EOL>if ret == singular:<EOL><INDENT>return singular_stripped<EOL><DEDENT>elif ret == plural:<EOL><INDENT>return plural_stripped<EOL><DEDENT>return ret<EOL>


class Translation(object):<EOL><INDENT>"""<STR_LIT>"""<EOL>ugettext = staticmethod(ugettext)<EOL>ungettext = staticmethod(ungettext)<EOL><DEDENT>import jingo<EOL>jingo.env.install_gettext_translations(Translation)<EOL>

Install our gettext and ngettext functions into Jinja2's environment.


if INSTALL_JINJA_TRANSLATIONS:<EOL><INDENT>install_jinja_translations()<EOL><DEDENT>if django.VERSION >= (<NUM_LIT:1>, <NUM_LIT:3>):<EOL><INDENT>django_trans._active.value = _activate(locale)<EOL><DEDENT>else:<EOL><INDENT>from django.utils.thread_support import currentThread<EOL>django_trans._active[currentThread()] = _activate(locale)<EOL><DEDENT>

Override django's utils.translation.activate().  Django forces files
to be named django.mo (http://code.djangoproject.com/ticket/6376).  Since
that's dumb and we want to be able to load different files depending on
what part of the site the user is in, we'll make our own function here.


django_trans.deactivate_all()<EOL>django_trans._translations = {}<EOL>

Override django's utils.translation.deactivate_all().  Django continues
    to cache a catalog even if you call their deactivate_all().


if isinstance(message, basestring):<EOL><INDENT>message = strip_whitespace(message)<EOL><DEDENT>elif isinstance(message, tuple):<EOL><INDENT>if len(message) == <NUM_LIT:2>:<EOL><INDENT>message = add_context(message[<NUM_LIT:1>], message[<NUM_LIT:0>])<EOL><DEDENT>elif len(message) == <NUM_LIT:3>:<EOL><INDENT>if all(isinstance(x, basestring) for x in message[:<NUM_LIT:2>]):<EOL><INDENT>singular, plural, num = message<EOL>message = (strip_whitespace(singular),<EOL>strip_whitespace(plural),<EOL>num)<EOL><DEDENT><DEDENT>elif len(message) == <NUM_LIT:4>:<EOL><INDENT>singular, plural, num, ctxt = message<EOL>message = (add_context(ctxt, strip_whitespace(singular)),<EOL>add_context(ctxt, strip_whitespace(plural)),<EOL>num)<EOL><DEDENT><DEDENT>return message<EOL>

We piggyback on jinja2's babel_extract() (really, Babel's extract_*
    functions) but they don't support some things we need so this function will
    tweak the message.  Specifically:

        1) We strip whitespace from the msgid.  Jinja2 will only strip
            whitespace from the ends of a string so linebreaks show up in
            your .po files still.

        2) Babel doesn't support context (msgctxt).  We hack that in ourselves
            here.


global static_stages<EOL>if static_stages is None:<EOL><INDENT>static_stages = PipelineStages()<EOL>stages = static_stages<EOL>if '<STR_LIT>' in config:<EOL><INDENT>path = config['<STR_LIT>']<EOL>if not os.path.isabs(path) and config.get('<STR_LIT>'):<EOL><INDENT>path = os.path.join(config['<STR_LIT>'], path)<EOL><DEDENT>try:<EOL><INDENT>stages.load_external_stages(config['<STR_LIT>'])<EOL><DEDENT>except IOError:<EOL><INDENT>return streamcorpus_pipeline  <EOL><DEDENT><DEDENT>if '<STR_LIT>' in config:<EOL><INDENT>for mod in config['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>stages.load_module_stages(mod)<EOL><DEDENT>except ImportError:<EOL><INDENT>return streamcorpus_pipeline  <EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>stages = static_stages<EOL><DEDENT>new_sub_modules = set(stage<EOL>for stage in stages.values()<EOL>if hasattr(stage, '<STR_LIT>'))<EOL>return NewSubModules(streamcorpus_pipeline, new_sub_modules)<EOL>

Replace the top-level pipeline configurable object.

    This investigates a number of sources, including
    `external_stages_path` and `external_stages_modules` configuration
    and `streamcorpus_pipeline.stages` entry points, and uses these to
    find the actual :data:`sub_modules` for
    :mod:`streamcorpus_pipeline`.


def __init__(self, kvl, hash_docs=True, hash_frequencies=True,<EOL>hash_keywords=True, keyword_tagger_ids=None,<EOL>keyword_size_limit=<NUM_LIT>):

self.client = kvl<EOL>self.hash_docs = hash_docs<EOL>self.hash_frequencies = hash_frequencies<EOL>self.hash_keywords = hash_keywords<EOL>self.keyword_tagger_ids = keyword_tagger_ids<EOL>self.keyword_size_limit = keyword_size_limit<EOL>

Create a new keyword indexer.

        `kvl` should be preconfigured to handle the
        :mod:`streamcorpus_pipeline` tables; this constructor will not
        call
        :meth:`~kvlayer._abstract_storage.AbstractStorage.setup_namespace`.
        The various `hash` parameters control what :meth:`index`
        writes out, and if only reading the default values will work.

        :param kvl: :mod:`kvlayer` client object
        :param hash_docs: if true, record hash to document ID index
        :param hash_frequencies: if true, record hash document frequencies
        :param hash_keywords: if true, record hash to keyword reverse index
        :param keyword_tagger_ids: if not :const:`None`, only consider
          these tagger IDs
        :param int keyword_size_limit: largest token to consider


(_, h) = self.make_hash_kw(tok)<EOL>return h<EOL>

Get a Murmur hash for a token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.


if isinstance(tok, unicode):<EOL><INDENT>tok = tok.encode('<STR_LIT:utf-8>')<EOL><DEDENT>h = mmh3.hash(tok)<EOL>if h == DOCUMENT_HASH_KEY:<EOL><INDENT>h = DOCUMENT_HASH_KEY_REPLACEMENT<EOL><DEDENT>return (tok, h)<EOL>

Get a Murmur hash and a normalized token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.

        :param tok: token to hash
        :return: pair of normalized `tok` and its hash


counter = Counter()<EOL>for tagger_id, sentences in si.body.sentences.iteritems():<EOL><INDENT>if ((self.keyword_tagger_ids is not None<EOL>and tagger_id not in self.keyword_tagger_ids)):<EOL><INDENT>continue<EOL><DEDENT>for sentence in sentences:<EOL><INDENT>for token in sentence.tokens:<EOL><INDENT>term = token.token  <EOL>term = term.decode('<STR_LIT:utf-8>')<EOL>term = cleanse(term)<EOL>if ((self.keyword_size_limit is not None and<EOL>len(term) > self.keyword_size_limit)):<EOL><INDENT>continue<EOL><DEDENT>if term not in self.stop_words:<EOL><INDENT>counter[term] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return counter<EOL>

Collect all of the words to be indexed from a stream item.

        This scans `si` for all of the configured tagger IDs.  It
        collects all of the token values (the
        :attr:`streamcorpus.Token.token`) and returns a
        :class:`collections.Counter` of them.

        :param si: stream item to scan
        :type si: :class:`streamcorpus.StreamItem`
        :return: counter of :class:`unicode` words to index
        :returntype: :class:`collections.Counter`


if not si.body.clean_visible:<EOL><INDENT>logger.warn('<STR_LIT>'<EOL>'<STR_LIT>', si.stream_id)<EOL>return<EOL><DEDENT>hash_counts = defaultdict(int)<EOL>hash_counts[DOCUMENT_HASH_KEY] = <NUM_LIT:1><EOL>hash_kw = defaultdict(int)<EOL>words = self.collect_words(si)<EOL>for tok, count in words.iteritems():<EOL><INDENT>(tok, tok_hash) = self.make_hash_kw(tok)<EOL>hash_counts[tok_hash] += count<EOL>hash_kw[tok] = tok_hash<EOL><DEDENT>if self.hash_docs:<EOL><INDENT>(k1, k2) = key_for_stream_item(si)<EOL>kvps = [((h, k1, k2), n) for (h, n) in hash_counts.iteritems()<EOL>if h != DOCUMENT_HASH_KEY]<EOL>self.client.put(HASH_TF_INDEX_TABLE, *kvps)<EOL><DEDENT>if self.hash_frequencies:<EOL><INDENT>kvps = [((h,), <NUM_LIT:1>) for h in hash_counts.iterkeys()]<EOL>self.client.increment(HASH_FREQUENCY_TABLE, *kvps)<EOL><DEDENT>if self.hash_keywords:<EOL><INDENT>kvps = [((h, t), <NUM_LIT:1>) for (t, h) in hash_kw.iteritems()]<EOL>self.client.increment(HASH_KEYWORD_INDEX_TABLE, *kvps)<EOL><DEDENT>

Record index records for a single document.

        Which indexes this creates depends on the parameters to the
        constructor.  This records all of the requested indexes for
        a single document.


return [tok_encoded.decode('<STR_LIT:utf8>')<EOL>for (_, tok_encoded) in<EOL>self.client.scan_keys(HASH_KEYWORD_INDEX_TABLE,<EOL>((tok_hash,), (tok_hash,)))]<EOL>

Get strings that correspond to some hash.

        No string will correspond to :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int tok_hash: Murmur hash to query
        :return: list of :class:`unicode` strings


result = {}<EOL>for (k, v) in self.client.get(HASH_FREQUENCY_TABLE,<EOL>*[(h,) for h in hashes]):<EOL><INDENT>if v is None:<EOL><INDENT>v = <NUM_LIT:0><EOL><DEDENT>result[k[<NUM_LIT:0>]] = v<EOL><DEDENT>return result<EOL>

Get document frequencies for a list of hashes.

        This will return all zeros unless the index was written with
        `hash_frequencies` set.  If :data:`DOCUMENT_HASH_KEY` is
        included in `hashes`, that value will be returned with the
        total number of documents indexed.  If you are looking for
        documents with that hash, pass
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param hashes: hashes to query
        :paramtype hashes: list of :class:`int`
        :return: map from hash to document frequency


for (_, k1, k2) in self.client.scan_keys(HASH_TF_INDEX_TABLE,<EOL>((h,), (h,))):<EOL><INDENT>yield kvlayer_key_to_stream_id((k1, k2))<EOL><DEDENT>

Get stream IDs for a single hash.

        This yields strings that can be retrieved using
        :func:`streamcorpus_pipeline._kvlayer.get_kvlayer_stream_item`,
        or fed back into :mod:`coordinate` or other job queue systems.

        Note that for common terms this can return a large number of
        stream IDs!  This is a scan over a dense region of a
        :mod:`kvlayer` table so it should be reasonably efficient,
        but be prepared for it to return many documents in a large
        corpus.  Blindly storing the results in a :class:`list`
        may be inadvisable.

        This will return nothing unless the index was written with
        :attr:`hash_docs` set.  No document will correspond to
        :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int h: Murmur hash to look up


for ((_, k1, k2), v) in self.client.scan(HASH_TF_INDEX_TABLE,<EOL>((h,), (h,))):<EOL><INDENT>yield (kvlayer_key_to_stream_id((k1, k2)), v)<EOL><DEDENT>

Get stream IDs and term frequencies for a single hash.

        This yields pairs of strings that can be retrieved using
        :func:`streamcorpus_pipeline._kvlayer.get_kvlayer_stream_item`
        and the corresponding term frequency.

        ..see:: :meth:`lookup`


def retry_func(self, *args, **kwargs):<EOL><INDENT>tries = <NUM_LIT:1><EOL>while True:<EOL><INDENT>try:<EOL><INDENT>return func(self, *args, **kwargs)<EOL>break<EOL><DEDENT>except OSError as exc:<EOL><INDENT>logger.error('<STR_LIT>')<EOL>raise<EOL><DEDENT>except FailedExtraction as exc:<EOL><INDENT>logger.error('<STR_LIT>', tries, exc_info=True)<EOL>raise<EOL><DEDENT>except FailedVerification as exc:<EOL><INDENT>logger.warn('<STR_LIT>', tries, exc_info=True)<EOL>if tries >= self.config['<STR_LIT>']:<EOL><INDENT>if self.config.get('<STR_LIT>'):<EOL><INDENT>logger.warn('<STR_LIT>')<EOL>break<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>logger.warn('<STR_LIT>', tries, exc_info=True)<EOL>if tries >= self.config['<STR_LIT>']:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>logger.warn('<STR_LIT>', self.config['<STR_LIT>'] - tries)<EOL>time.sleep(<NUM_LIT:3> * tries)<EOL>tries += <NUM_LIT:1><EOL><DEDENT><DEDENT>return retry_func<EOL>

Decorator for methods that need many retries, because of
intermittent failures, such as AWS calls via boto, which has a
non-back-off retry.


def verify_md5(md5_expected, data, other_errors=None):

<EOL>)<EOL>!= %r = received md5' \ d5_recv))<EOL>

return True if okay, raise Exception if not


if not bucket_name:<EOL><INDENT>if '<STR_LIT>' not in config:<EOL><INDENT>raise ConfigurationError(<EOL>'<STR_LIT>')<EOL><DEDENT>bucket_name = config['<STR_LIT>']<EOL><DEDENT>aws_access_key_id_path = config.get('<STR_LIT>')<EOL>aws_secret_access_key_path = config.get('<STR_LIT>')<EOL>params = ()<EOL>if aws_access_key_id_path and aws_secret_access_key_path:<EOL><INDENT>try:<EOL><INDENT>access = open(aws_access_key_id_path).read().strip()<EOL>secret = open(aws_secret_access_key_path).read().strip()<EOL>params = (access, secret)<EOL><DEDENT>except:<EOL><INDENT>logger.error('<STR_LIT>', exc_info=True)<EOL>raise<EOL><DEDENT><DEDENT>conn = S3Connection(*params)<EOL>bucket = conn.get_bucket(bucket_name)<EOL>return bucket<EOL>

This function is mostly about managing configuration, and then
    finally returns a boto.Bucket object.

    AWS credentials come first from config keys
    aws_access_key_id_path, aws_secret_access_key_path (paths to one
    line files); secondly from environment variables
    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY; also from $HOME/.aws/credentials
    or the magic Amazon http://169.254.169.254/ service.  If credentials
    are not set in the config then behavior is the same as other AWS-based
    command-line tools.


if i_str.startswith('<STR_LIT>'):<EOL><INDENT>bucket_name, kpath = i_str[<NUM_LIT:5>:].split('<STR_LIT:/>', <NUM_LIT:1>)<EOL>assert bucket_name and kpath, '<STR_LIT>'.format(i_str)<EOL><DEDENT>else:<EOL><INDENT>bucket_name = None<EOL>kpath = os.path.join(self.config['<STR_LIT>'].strip(),<EOL>i_str.strip())<EOL><DEDENT>logger.info('<STR_LIT>', bucket_name, kpath)<EOL>return self.get_chunk(bucket_name, kpath)<EOL>

Takes a path suffix as a string over stdin and generates chunks from
s3://<bucket><s3_prefix_path>/{i_str}.


informat = self.config['<STR_LIT>'].lower()<EOL>if informat == '<STR_LIT>':<EOL><INDENT>return _generate_stream_items(data)<EOL><DEDENT>elif informat == '<STR_LIT>':<EOL><INDENT>ver = self.config['<STR_LIT>']<EOL>if ver not in _message_versions:<EOL><INDENT>raise ConfigurationError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (ver, '<STR_LIT:U+002CU+0020>'.join(list(_message_versions.keys()))))<EOL><DEDENT>message = _message_versions[ver]<EOL>return streamcorpus.Chunk(data=data, message=message)<EOL><DEDENT>elif informat == '<STR_LIT>' and FCChunk is not None:<EOL><INDENT>return FCChunk(data=data)<EOL><DEDENT>else:<EOL><INDENT>raise ConfigurationError(<EOL>'<STR_LIT>'<EOL>% informat)<EOL><DEDENT>

Given the raw data from s3, return a generator for the items
contained in that data. A generator is necessary to support
chunk files, but non-chunk files can be provided by a generator
that yields exactly one item.

Decoding works by case analysis on the config option
``input_format``. If an invalid ``input_format`` is given, then
a ``ConfigurationError`` is raised.


@_retry<EOL><INDENT>def get_chunk(self, bucket_name, key_path):<DEDENT>

bucket = get_bucket(self.config, bucket_name=bucket_name)<EOL>key = bucket.get_key(key_path)<EOL>if key is None:<EOL><INDENT>raise FailedExtraction('<STR_LIT>' % key_path)<EOL><DEDENT>fh = StringIO()<EOL>key.get_contents_to_file(fh)<EOL>data = fh.getvalue()<EOL>if not data:<EOL><INDENT>raise FailedExtraction('<STR_LIT>'<EOL>% key.key)<EOL><DEDENT>chunk_type, compression, encryption = parse_file_extensions(key_path)<EOL>if encryption == '<STR_LIT>':<EOL><INDENT>if not self.gpg_decryption_key_path:<EOL><INDENT>raise FailedExtraction('<STR_LIT>'<EOL>% (key.key, self.gpg_decryption_key_path))<EOL><DEDENT><DEDENT>_errors = []<EOL>if compression or encryption:<EOL><INDENT>_errors, data = decrypt_and_uncompress(<EOL>data, <EOL>self.gpg_decryption_key_path,<EOL>tmp_dir=self.config.get('<STR_LIT>'),<EOL>compression=compression,<EOL>)<EOL>if not data:<EOL><INDENT>msg = '<STR_LIT>'+ '<STR_LIT>'.format(key_path, len(data), '<STR_LIT:\n>'.join(_errors))<EOL>logger.error(msg)<EOL>raise FailedExtraction(msg)<EOL><DEDENT>logger.info( '<STR_LIT:\n>'.join(_errors) )<EOL><DEDENT>if not self.config['<STR_LIT>']:<EOL><INDENT>logger.warn('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>logger.info('<STR_LIT>' % key.key)<EOL>m = re.search('<STR_LIT>', key.key)<EOL>if m is None:<EOL><INDENT>raise FailedExtraction(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% key.key)<EOL><DEDENT>i_content_md5 = m.group(<NUM_LIT:1>)<EOL>verify_md5(i_content_md5, data, other_errors=_errors)<EOL><DEDENT>return self._decode(data)<EOL>

return Chunk object full of records
        bucket_name may be None


def __call__(self, t_path, name_info, i_str):

logger.info('<STR_LIT>',<EOL>t_path, name_info, i_str)<EOL>try:<EOL><INDENT>more_name_info = get_name_info(t_path, i_str=i_str,<EOL>chunk_type=self.chunk_type)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.critical('<STR_LIT>', t_path, i_str, exc_info=True)<EOL>raise<EOL><DEDENT>self.name_info = dict(name_info, **more_name_info)<EOL>if self.name_info['<STR_LIT>'] == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>o_path = self.s3key_name<EOL>logger.info('<STR_LIT>',<EOL>self.__class__.__name__, o_path, i_str, t_path)<EOL>t_path2 = self.prepare_on_disk(t_path)<EOL>data_len = os.path.getsize(t_path2)<EOL>if data_len == <NUM_LIT:0>:<EOL><INDENT>logger.critical('<STR_LIT>')<EOL><DEDENT>logger.debug('<STR_LIT>', data_len, t_path2)<EOL>self.put_data(o_path, t_path2, self.name_info['<STR_LIT>'], data_len)<EOL>self.cleanup(t_path, t_path2)<EOL>logger.info('<STR_LIT>',<EOL>self.__class__.__name__, i_str, o_path)<EOL>return [o_path]<EOL>

Load chunk from t_path and put it into the right place in s3
using the output_name template from the config


try:<EOL><INDENT>parts = urlparse(raw_string)<EOL>domain = parts.netloc.split('<STR_LIT::>')[<NUM_LIT:0>]<EOL><DEDENT>except:<EOL><INDENT>domain = '<STR_LIT>'<EOL><DEDENT>if not domain:<EOL><INDENT>domain = raw_string<EOL><DEDENT>if not domain:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>domain = re.sub('<STR_LIT>', '<STR_LIT>', domain.strip().lower())<EOL>return domain<EOL>

extract a lower-case, no-slashes domain name from a raw string
    that might be a URL


cuts = []<EOL>if domain:<EOL><INDENT>parts = domain.split('<STR_LIT:.>')<EOL>for i in range(len(parts)):<EOL><INDENT>cuts.append( '<STR_LIT:.>'.join(parts[i:]))<EOL><DEDENT><DEDENT>return cuts<EOL>

returns a list of strings created by splitting the domain on
    '.' and successively cutting off the left most portion


parts = []<EOL>st = master.status(name)<EOL>a = st['<STR_LIT>']<EOL>if a > <NUM_LIT:0>:<EOL><INDENT>parts.append("<STR_LIT>".format(a))<EOL><DEDENT>p = st['<STR_LIT>']<EOL>if p > <NUM_LIT:0>:<EOL><INDENT>parts.append("<STR_LIT>".format(p))<EOL><DEDENT>f = st['<STR_LIT>']<EOL>if f > <NUM_LIT:0>:<EOL><INDENT>parts.append("<STR_LIT>".format(f))<EOL><DEDENT>z = st['<STR_LIT>']<EOL>if z > <NUM_LIT:0>:<EOL><INDENT>parts.append("<STR_LIT>".format(z))<EOL><DEDENT>if len(parts) == <NUM_LIT:0>:<EOL><INDENT>parts.append("<STR_LIT>")<EOL><DEDENT>return '<STR_LIT:U+002CU+0020>'.join(parts)<EOL>

Get a quick summary of the status of 'name' as a (printable)
    string.


return '<STR_LIT>'.join(['<STR_LIT>'.format(j, job_status(master, j))<EOL>for j in jobs])<EOL>

Get a quick summary of the statuses of all of 'jobs' as a
    (printable) string.


ext = '<STR_LIT>' if chunker['<STR_LIT>'].lower() == '<STR_LIT>' else '<STR_LIT>'<EOL>for k in dummies.keys():<EOL><INDENT>new_k = '<STR_LIT:.>'.join([k, ext, '<STR_LIT>'])<EOL>dummies[new_k] = dummies.pop(k)<EOL><DEDENT>

now that to_s3_chunks uses the file extension to detect the
    compression and type, this test has to make real filenames with
    proper extesions.


reader = ProtoStreamReader(f)<EOL>return filter(<EOL>lambda x: x is not None,<EOL>map(_make_stream_item, reader))<EOL>

Given a spinn3r feed, produce a sequence of valid StreamItems.

    Because of goopy Python interactions, you probably need to call
    this and re-yield its results, as

    >>> with open(filename, 'rb') as f:
    ...   for si in _make_stream_items(f):
    ...     yield si


<EOL>if not hasattr(entry, '<STR_LIT>'):<EOL><INDENT>return None<EOL><DEDENT>pe = entry.permalink_entry<EOL>si = streamcorpus.make_stream_item(<EOL>pe.date_found[:-<NUM_LIT:1>] + '<STR_LIT>',<EOL>pe.canonical_link.href.encode('<STR_LIT:utf8>'))<EOL>if not si.stream_time:<EOL><INDENT>logger.debug('<STR_LIT>'<EOL>.format(pe.date_found))<EOL>return None<EOL><DEDENT>if not si.abs_url:<EOL><INDENT>logger.debug('<STR_LIT>'<EOL>.format(pe.canonical_link.href))<EOL>return None<EOL><DEDENT>si.body = _make_content_item(<EOL>pe.content,<EOL>alternate_data=entry.feed_entry.content.data)<EOL>if not si.body:<EOL><INDENT>return None<EOL><DEDENT>if not si.body.raw:<EOL><INDENT>return None<EOL><DEDENT>if pe.content_extract.data:<EOL><INDENT>si.other_content['<STR_LIT>'] = _make_content_item(pe.content_extract)<EOL><DEDENT>si.other_content['<STR_LIT:title>'] = streamcorpus.ContentItem(<EOL>raw=pe.title.encode('<STR_LIT:utf8>'),<EOL>media_type=pe.content_extract.mime_type,<EOL>encoding='<STR_LIT>')<EOL>si.other_content['<STR_LIT>'] = streamcorpus.ContentItem(<EOL>raw=entry.feed_entry.title.encode('<STR_LIT:utf8>'),<EOL>media_type=entry.feed_entry.content.mime_type,<EOL>encoding='<STR_LIT>')<EOL>if entry.feed_entry.content.data:<EOL><INDENT>si.other_content['<STR_LIT>'] = _make_content_item(<EOL>entry.feed_entry.content)<EOL><DEDENT>si.source_metadata['<STR_LIT>'] = pe.lang[<NUM_LIT:0>].code<EOL>si.source_metadata['<STR_LIT>'] = json.dumps(<EOL>dict(<EOL>name=pe.author[<NUM_LIT:0>].name,<EOL>email=pe.author[<NUM_LIT:0>].email,<EOL>link=pe.author[<NUM_LIT:0>].link[<NUM_LIT:0>].href,<EOL>)<EOL>)<EOL>si.source = entry.source.publisher_type<EOL>return si<EOL>

Given a single spinn3r feed entry, produce a single StreamItem.

    Returns 'None' if a complete item can't be constructed.


def _make_content_item(node, mime_type=None, alternate_data=None):

raw = node.data<EOL>if getattr(node, '<STR_LIT>', None) == '<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>raw = zlib.decompress(node.data)<EOL><DEDENT>except Exception as exc:<EOL><INDENT>if alternate_data is not None:<EOL><INDENT>try:<EOL><INDENT>raw = zlib.decompress(alternate_data)<EOL><DEDENT>except Exception:<EOL><INDENT>raise exc  <EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if mime_type is None:<EOL><INDENT>mime_type = node.mime_type<EOL><DEDENT>raw = raw.decode('<STR_LIT:utf8>').encode('<STR_LIT:utf8>')<EOL>return streamcorpus.ContentItem(raw=raw, media_type=mime_type)<EOL>

Create a ContentItem from a node in the spinn3r data tree.

    The ContentItem is created with raw data set to ``node.data``,
    decompressed if the node's encoding is 'zlib', and UTF-8
    normalized, with a MIME type from ``node.mime_type``.

    ``node``
      the actual node from the spinn3r protobuf data
    ``mime_type``
      string MIME type to use (defaults to ``node.mime_type``)
    ``alternate_data``
      alternate (compressed) data to use, if ``node.data`` is missing
      or can't be decompressed


def __init__(self, f, entry_type=Entry):

super(ProtoStreamReader, self).__init__()<EOL>self.f = f<EOL>self._prefix = '<STR_LIT>'<EOL>self._header = None<EOL>self._entry_type = entry_type<EOL>

Create a new ProtoStreamReader.

        ``f``
          file-like object to read
        ``entry_type``
          type of objects in this feed


self._prefix = b + self._prefix<EOL>

Push byte array 'b' on to the front of the prefix list; this
        undoes part of a _read().  This bytes will be the next to read.


if n <= len(self._prefix):<EOL><INDENT>result = self._prefix[:n]<EOL>self._prefix = self._prefix[n:]<EOL>return result<EOL><DEDENT>n -= len(self._prefix)<EOL>result = self._prefix + self.f.read(n)<EOL>self._prefix = "<STR_LIT>"<EOL>return result<EOL>

Read (up to) 'n' bytes from the underlying file.  If any bytes
        have been pushed in with _unread() those are returned first.


buf = self._read(<NUM_LIT:8>)<EOL>(n, l) = _DecodeVarint(buf, <NUM_LIT:0>)<EOL>self._unread(buf[l:])<EOL>return n<EOL>

Read exactly a varint out of the underlying file.


l = self._read_varint()<EOL>return self._read(l)<EOL>

Read a length from the file, then read that many bytes.
        Return the read block.


o = cls()<EOL>o.ParseFromString(self._read_block())<EOL>return o<EOL>

Read some protobuf-encoded object stored in a single block
        out of the file.


self._header = self._read_a(ProtoStreamHeader)<EOL>

Read the header block out of the underlying file.
        After this (and forevermore afterwards) the file will point
        at the start of a ProtoStreamDelimiter block.


if self._header is None:<EOL><INDENT>self._bootstrap()<EOL><DEDENT>while True:<EOL><INDENT>delim = self._read_a(ProtoStreamDelimiter)<EOL>if delim.delimiter_type == ProtoStreamDelimiter.END:<EOL><INDENT>return<EOL><DEDENT>assert delim.delimiter_type == ProtoStreamDelimiter.ENTRY<EOL>yield self._read_a(self._entry_type)<EOL><DEDENT>

Iterate through the objects in the file.


if self._child:<EOL><INDENT>try:<EOL><INDENT>self._child.terminate()<EOL><DEDENT>except OSError as exc:<EOL><INDENT>if exc.errno == <NUM_LIT:3>:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>

send SIGTERM to the tagger child process


emails = bracket_emails.findall(text)<EOL>keys = []<EOL>for email in emails:<EOL><INDENT>_email = email.replace("<STR_LIT:<>", "<STR_LIT>").replace("<STR_LIT:>>", "<STR_LIT>")<EOL>text = text.replace(email, _email)<EOL><DEDENT>return text<EOL>

Replace all angle bracket emails with a unique key.


<EOL>parts = stream_id.split('<STR_LIT:->')<EOL>if len(parts) != <NUM_LIT:2>:<EOL><INDENT>raise KeyError('<STR_LIT>' + stream_id)<EOL><DEDENT>epoch_ticks_s = parts[<NUM_LIT:0>]<EOL>doc_id_s = parts[<NUM_LIT:1>]<EOL>if not epoch_ticks_s.isdigit():<EOL><INDENT>raise KeyError('<STR_LIT>' + stream_id)<EOL><DEDENT>if doc_id_s.lstrip(string.hexdigits) != '<STR_LIT>':<EOL><INDENT>raise KeyError('<STR_LIT>' + stream_id)<EOL><DEDENT>return (base64.b16decode(doc_id_s.upper()), int(epoch_ticks_s))<EOL>

Convert a text stream ID to a kvlayer key.

    The return tuple can be used directly as a key in the
    :data:`STREAM_ITEMS_TABLE` table.

    :param str stream_id: stream ID to convert
    :return: :mod:`kvlayer` key tuple
    :raise exceptions.KeyError: if `stream_id` is malformed


abs_url_hash, epoch_ticks = k<EOL>return '<STR_LIT>'.format(epoch_ticks,<EOL>base64.b16encode(abs_url_hash).lower())<EOL>

Convert a kvlayer key to a text stream ID.

    `k` should be of the same form produced by
    :func:`stream_id_to_kvlayer_key`.

    :param k: :mod:`kvlayer` key tuple
    :return: converted stream ID
    :returntype str:


<EOL>urlhash = hashlib.md5(si.abs_url).digest()<EOL>return (urlhash, int(si.stream_time.epoch_ticks))<EOL>

Get a kvlayer key from a stream item.

    The return tuple can be used directly as a key in the
    :data:`STREAM_ITEMS_TABLE` table.  Note that this recalculates the
    stream ID, and if the internal data on the stream item is inconsistent
    then this could return a different result from
    :func:`stream_id_to_kvlayer_key`.

    :param si: stream item to get key for
    :return: :mod:`kvlayer` key tuple


def html_entities_to_unicode(text, space_padding=False, safe_only=False):

def convert_entities(match):<EOL><INDENT>'''<STR_LIT>'''<EOL>x = match.group(<NUM_LIT:1>)<EOL>if safe_only and x not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD:<EOL><INDENT>return '<STR_LIT>' % x<EOL><DEDENT>if x in name2codepoint:<EOL><INDENT>return chr(name2codepoint[x])<EOL><DEDENT>elif x in XML_ENTITIES_TO_SPECIAL_CHARS:<EOL><INDENT>return XML_ENTITIES_TO_SPECIAL_CHARS[x]<EOL><DEDENT>elif len(x) > <NUM_LIT:0> and x[<NUM_LIT:0>] == '<STR_LIT:#>':<EOL><INDENT>if len(x) > <NUM_LIT:1> and x[<NUM_LIT:1>] == '<STR_LIT:x>':<EOL><INDENT>return chr(int(x[<NUM_LIT:2>:], <NUM_LIT:16>))<EOL><DEDENT>else:<EOL><INDENT>return chr(int(x[<NUM_LIT:1>:]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' % x<EOL><DEDENT><DEDENT>def convert_to_padded_entitites(match):<EOL><INDENT>converted_string = convert_entities(match)<EOL>num_spaces_needed = len(match.group(<NUM_LIT:0>)) - len(converted_string)<EOL>assert num_spaces_needed >= <NUM_LIT:0>,'<STR_LIT>' % (converted_string, match.group(<NUM_LIT:0>))<EOL>num_left = int(num_spaces_needed / <NUM_LIT:2>)<EOL>num_right = num_spaces_needed - num_left<EOL>return ('<STR_LIT:U+0020>' * num_left) + converted_string + ('<STR_LIT:U+0020>' * num_right)<EOL><DEDENT>if space_padding:<EOL><INDENT>return tags.sub(<EOL>convert_to_padded_entitites,<EOL>text)<EOL><DEDENT>else:<EOL><INDENT>return tags.sub(<EOL>convert_entities,<EOL>text)<EOL><DEDENT>

Convert any HTML, XML, or numeric entities in the attribute values.
For example '&' becomes '&'.

This is adapted from BeautifulSoup, which should be able to do the
same thing when called like this --- but this fails to convert
everything for some bug.

text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))


text = cgi.escape(text).encode('<STR_LIT:ascii>', '<STR_LIT>')<EOL>return text<EOL>

Converts unicode to HTML entities.  For example '&' becomes '&'.


def sentences_to_xpaths(sentences):<EOL><INDENT>tokens = sentences_to_char_tokens(sentences)<EOL>offsets = char_tokens_to_char_offsets(tokens)<EOL>return char_offsets_to_xpaths(html, offsets)<EOL><DEDENT>def xprange_to_offset(xprange):<EOL><INDENT>return Offset(type=OffsetType.XPATH_CHARS,<EOL>first=xprange.start_offset, length=<NUM_LIT:0>,<EOL>xpath=xprange.start_xpath,<EOL>content_form='<STR_LIT>', value=None,<EOL>xpath_end=xprange.end_xpath,<EOL>xpath_end_offset=xprange.end_offset)<EOL><DEDENT>html = unicode(si.body.clean_html, '<STR_LIT:utf-8>')<EOL>for sentences in si.body.sentences.itervalues():<EOL><INDENT>tokens = sentences_to_char_tokens(sentences)<EOL>for token, xprange in izip(tokens, sentences_to_xpaths(sentences)):<EOL><INDENT>if xprange is None:<EOL><INDENT>continue<EOL><DEDENT>offset = xprange_to_offset(xprange)<EOL>token.offsets[OffsetType.XPATH_CHARS] = offset<EOL><DEDENT><DEDENT>

Mutably tag tokens with xpath offsets.

    Given some stream item, this will tag all tokens from all taggings
    in the document that contain character offsets. Note that some
    tokens may not have computable xpath offsets, so an xpath offset
    for those tokens will not be set. (See the documentation and
    comments for ``char_offsets_to_xpaths`` for what it means for a
    token to have a computable xpath.)

    If a token can have its xpath offset computed, it is added to its
    set of offsets with a ``OffsetType.XPATH_CHARS`` key.


for sentence in si_sentences:<EOL><INDENT>for token in sentence.tokens:<EOL><INDENT>if OffsetType.CHARS in token.offsets:<EOL><INDENT>yield token<EOL><DEDENT><DEDENT><DEDENT>

Convert stream item sentences to character ``Offset``s.


for token in si_tokens:<EOL><INDENT>offset = token.offsets[OffsetType.CHARS]<EOL>yield offset.first, offset.first + offset.length<EOL><DEDENT>

Convert character ``Offset``s to character ranges.


html = uni(html)<EOL>parser = XpathTextCollector()<EOL>prev_end = <NUM_LIT:0><EOL>prev_progress = True<EOL>for start, end in char_offsets:<EOL><INDENT>if start == end:<EOL><INDENT>yield None<EOL>continue<EOL><DEDENT>if not prev_progress:<EOL><INDENT>for i in xrange(prev_end, start):<EOL><INDENT>parser.feed(html[i])<EOL>prev_end += <NUM_LIT:1><EOL>if parser.made_progress:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not parser.made_progress:<EOL><INDENT>yield None<EOL>continue<EOL><DEDENT><DEDENT>if prev_end < start:<EOL><INDENT>parser.feed(html[prev_end:start])<EOL>if not parser.made_progress:<EOL><INDENT>parser.feed(html[start:end])<EOL>prev_progress = parser.made_progress<EOL>prev_end = end<EOL>yield None<EOL>continue<EOL><DEDENT><DEDENT>xstart = parser.xpath_offset()<EOL>parser.feed(html[start:end])<EOL>xend = parser.xpath_offset()<EOL>prev_end = end<EOL>if not parser.made_progress:<EOL><INDENT>prev_progress = False<EOL>yield None<EOL><DEDENT>else:<EOL><INDENT>prev_progress = True<EOL>yield XpathRange(xstart[<NUM_LIT:0>], xstart[<NUM_LIT:1>], xend[<NUM_LIT:0>], xend[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>parser.feed(html[prev_end:])<EOL>parser.close()<EOL>

Converts HTML and a sequence of char offsets to xpath offsets.

    Returns a generator of :class:`streamcorpus.XpathRange` objects
    in correspondences with the sequence of ``char_offsets`` given.
    Namely, each ``XpathRange`` should address precisely the same text
    as that ``char_offsets`` (sans the HTML).

    Depending on how ``char_offsets`` was tokenized, it's possible that
    some tokens cannot have their xpaths generated reliably. In this
    case, a ``None`` value is yielded instead of a ``XpathRange``.

    ``char_offsets`` must be a sorted and non-overlapping sequence of
    character ranges. They do not have to be contiguous.


def debug(s):<EOL><INDENT>logger.warning(s)<EOL><DEDENT>def print_window(token, size=<NUM_LIT:200>):<EOL><INDENT>coffset = token.offsets[OffsetType.CHARS]<EOL>start = max(<NUM_LIT:0>, coffset.first - size)<EOL>end = min(len(html), coffset.first + coffset.length + size)<EOL>debug('<STR_LIT:->' * <NUM_LIT>)<EOL>debug(coffset)<EOL>debug('<STR_LIT>' % size)<EOL>debug(html[start:end])<EOL>debug('<STR_LIT:->' * <NUM_LIT>)<EOL><DEDENT>def debug_all(token, xprange, expected, err=None, got=None):<EOL><INDENT>debug('<STR_LIT:->' * <NUM_LIT>)<EOL>if err is not None:<EOL><INDENT>debug(err)<EOL><DEDENT>debug(xprange)<EOL>debug('<STR_LIT>' % expected)<EOL>if got is not None:<EOL><INDENT>debug('<STR_LIT>' % got)<EOL><DEDENT>debug('<STR_LIT>' % unicode(token.token, '<STR_LIT:utf-8>'))<EOL>print_window(token, size=<NUM_LIT:10>)<EOL>print_window(token, size=<NUM_LIT:30>)<EOL>print_window(token, size=<NUM_LIT:100>)<EOL>print_window(token, size=<NUM_LIT:200>)<EOL>debug('<STR_LIT:->' * <NUM_LIT>)<EOL><DEDENT>def slice_clean_visible(token):<EOL><INDENT>coffset = token.offsets[OffsetType.CHARS]<EOL>return cleanvis[coffset.first:coffset.first + coffset.length]<EOL><DEDENT>def test_token(token):<EOL><INDENT>coffset = token.offsets.get(OffsetType.CHARS)<EOL>if coffset is None:<EOL><INDENT>return False<EOL><DEDENT>xoffset = token.offsets.get(OffsetType.XPATH_CHARS)<EOL>if xoffset is None:<EOL><INDENT>return False<EOL><DEDENT>crange = (coffset.first, coffset.first + coffset.length)<EOL>xprange = XpathRange.from_offset(xoffset)<EOL>expected = slice_clean_visible(token)<EOL>if expected != unicode(token.token, '<STR_LIT:utf-8>'):<EOL><INDENT>return False<EOL><DEDENT>try:<EOL><INDENT>got = xprange.slice_node(html_root)<EOL><DEDENT>except InvalidXpathError as err:<EOL><INDENT>debug_all(token, xprange, expected, err=err)<EOL>raise XpathMismatchError(html, cleanvis, xprange, crange)<EOL><DEDENT>if expected != got:<EOL><INDENT>debug_all(token, xprange, expected, got=got)<EOL>raise XpathMismatchError(html, cleanvis, xprange, crange)<EOL><DEDENT>return True<EOL><DEDENT>cleanvis = unicode(si.body.clean_visible, '<STR_LIT:utf-8>')<EOL>html = unicode(si.body.clean_html, '<STR_LIT:utf-8>')<EOL>html_root = XpathRange.html_node(html)<EOL>total, has_valid_xpath = <NUM_LIT:0>, <NUM_LIT:0><EOL>for sentences in si.body.sentences.itervalues():<EOL><INDENT>for sentence in sentences:<EOL><INDENT>if quick:<EOL><INDENT>for i in xrange(len(sentence.tokens) - <NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>if test_token(sentence.tokens[i]):<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for token in sentence.tokens:<EOL><INDENT>total += <NUM_LIT:1><EOL>if test_token(token):<EOL><INDENT>has_valid_xpath += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if not quick:<EOL><INDENT>logger.info('<STR_LIT>',<EOL>si.stream_id, has_valid_xpath, total)<EOL><DEDENT>

Roundtrip all Xpath offsets in the given stream item.

    For every token that has both ``CHARS`` and ``XPATH_CHARS``
    offsets, slice the ``clean_html`` with the ``XPATH_CHARS`` offset
    and check that it matches slicing ``clean_visible`` with the
    ``CHARS`` offset.

    If this passes without triggering an assertion, then we're
    guaranteed that all ``XPATH_CHARS`` offsets in the stream item are
    correct. (Note that does not check for completeness. On occasion, a
    token's ``XPATH_CHARS`` offset cannot be computed.)

    There is copious debugging output to help make potential bugs
    easier to track down.

    This is used in tests in addition to the actual transform. It's
    expensive to run, but not running it means silent and hard to
    debug bugs.


<EOL>if tag is TextElement and self.last_tag is TextElement:<EOL><INDENT>return<EOL><DEDENT>self.last_tag = tag<EOL>if tag not in self.tags:<EOL><INDENT>self.tags[tag] = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self.tags[tag] += <NUM_LIT:1><EOL><DEDENT>

Record that `tag` has been seen at this depth.

        If `tag` is :class:`TextElement`, it records a text node.


if self.last_tag is TextElement:<EOL><INDENT>return '<STR_LIT>'.format(count=self.text_index())<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'.format(tag=self.last_tag,<EOL>count=self.tags[self.last_tag])<EOL><DEDENT>

Get an XPath fragment for this location.

        It is of the form ``tag[n]`` where `tag` is the most recent
        element added and n is its position.


<EOL>i = self.tags.get(TextElement, <NUM_LIT:0>)<EOL>if self.last_tag is not TextElement:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>return i<EOL>

Returns the one-based index of the current text node.


datai = self.depth_stack[-<NUM_LIT:1>].text_index()<EOL>xpath = (u'<STR_LIT:/>' +<EOL>u'<STR_LIT:/>'.join(dse.xpath_piece()<EOL>for dse in self.depth_stack[:-<NUM_LIT:1>]) +<EOL>(u'<STR_LIT>' % datai))<EOL>return (xpath, self.data_start)<EOL>

Returns a tuple of ``(xpath, character offset)``.

        The ``xpath`` returned *uniquely* identifies the end of the
        text node most recently inserted. The character offsets
        indicates where the text inside the node ends. (When the text
        node is empty, the offset returned is `0`.)


def tps(text, min_token_len=<NUM_LIT:2>, quant_rate=<NUM_LIT>):

counts = Counter(<EOL>ifilter(lambda x: len(x) >= min_token_len, <EOL>imap(cleanse, text.split())))<EOL>max_freq = counts.most_common(<NUM_LIT:1>)[<NUM_LIT:0>][<NUM_LIT:1>]<EOL>if max_freq <= <NUM_LIT:1>:<EOL><INDENT>quant = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>quant = max(<NUM_LIT:2>, round(max_freq * quant_rate))<EOL><DEDENT>to_hash = []<EOL>for word, count in counts.most_common():<EOL><INDENT>if count <= quant:<EOL><INDENT>break<EOL><DEDENT>to_hash += [word, str(int(math.floor(count / quant)))]<EOL><DEDENT>to_hash = u'<STR_LIT:U+0020>'.join(to_hash)<EOL>return hashlib.md5(to_hash.encode('<STR_LIT:utf8>')).hexdigest()<EOL>

:param text: tag-free UTF-8 string of free text
:returns string: 32-character md5 hash in hexadecimal

Python implementation of the TextProfileSignature provided in
SOLR.  Unlike most other locality sensitive hashes, a TPS can be
indexed as a searchable property of each document that does not
require n^2 comparisons to find duplicates.

http://wiki.apache.org/solr/TextProfileSignature


text = '<STR_LIT>'<EOL>html = fix_emails(html)<EOL>for m in invisible.finditer(html):<EOL><INDENT>text += m.group('<STR_LIT>')<EOL>text += '<STR_LIT:U+0020>' * len(m.group('<STR_LIT>'))<EOL><DEDENT>assert len(html) >= len(text), '<STR_LIT>' % (len(html), len(text))<EOL>tail = len(html) - len(text)<EOL>text += html[-tail:]<EOL>assert len(html) == len(text), '<STR_LIT>' % (len(html), len(text))<EOL>return text<EOL>

Takes an HTML-like binary string as input and returns a binary
string of the same length with all tags replaced by whitespace.
This also detects script and style tags, and replaces the text
between them with whitespace.

Pre-existing whitespace of any kind (newlines, tabs) is converted
to single spaces ' ', which has the same byte length (and
character length).

Note: this does not change any characters like ’ and  ,
so taggers operating on this text must cope with such symbols.
Converting them to some other character would change their byte
length, even if equivalent from a character perspective.

This is regex based, which can occassionally just hang...


def non_tag_chars(html):<EOL><INDENT>n = <NUM_LIT:0><EOL>while n < len(html):<EOL><INDENT>angle = html.find('<STR_LIT:<>', n)<EOL>if angle == -<NUM_LIT:1>:<EOL><INDENT>yield html[n:]<EOL>n = len(html)<EOL>break<EOL><DEDENT>yield html[n:angle]<EOL>n = angle<EOL>while n < len(html):<EOL><INDENT>nl = html.find('<STR_LIT:\n>', n)<EOL>angle = html.find('<STR_LIT:>>', n)<EOL>if angle == -<NUM_LIT:1>:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (len(html) - n)<EOL>n = len(html)<EOL>break<EOL><DEDENT>elif nl == -<NUM_LIT:1> or angle < nl:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (angle + <NUM_LIT:1> - n)<EOL>n = angle + <NUM_LIT:1><EOL>break<EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (nl - n) + '<STR_LIT:\n>'<EOL>n = nl + <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>if not isinstance(_html, str):<EOL><INDENT>_html = str(_html, '<STR_LIT:utf-8>')<EOL><DEDENT>_html = fix_emails(_html)<EOL>non_tag = '<STR_LIT>'.join(non_tag_chars(_html))<EOL>return non_tag.encode('<STR_LIT:utf-8>')<EOL>

Takes an HTML-like Unicode string as input and returns a UTF-8
encoded string with all tags replaced by whitespace. In particular,
all Unicode characters inside HTML are replaced with a single
whitespace character.

This does not detect comments, style, script, link.  It also does
do anything with HTML-escaped characters.  All of these are
handled by the clean_html pre-cursor step.

Pre-existing whitespace of any kind (newlines, tabs) is converted
to single spaces ' ', which has the same byte length (and
character length).

This is a simple state machine iterator without regexes


n = <NUM_LIT:0><EOL>while n < len(html):<EOL><INDENT>angle = html.find('<STR_LIT:<>', n)<EOL>if angle == -<NUM_LIT:1>:<EOL><INDENT>yield html[n:]<EOL>n = len(html)<EOL>break<EOL><DEDENT>yield html[n:angle]<EOL>n = angle<EOL>space = html.find('<STR_LIT:U+0020>',  n, n + longest_extended_tag + <NUM_LIT:2>)<EOL>angle = html.find('<STR_LIT:>>',  n, n + longest_extended_tag + <NUM_LIT:2>)<EOL>nl    = html.find('<STR_LIT:\n>', n, n + longest_extended_tag + <NUM_LIT:2>)<EOL>tab   = html.find('<STR_LIT:\t>', n, n + longest_extended_tag + <NUM_LIT:2>)<EOL>ends = [end for end in [tab, nl, space, angle] if end > -<NUM_LIT:1>]<EOL>if ends:<EOL><INDENT>tag = html[n + <NUM_LIT:1> : min(ends)]<EOL>if tag == '<STR_LIT>':<EOL><INDENT>end = html.find('<STR_LIT>', n)<EOL>while n < end:<EOL><INDENT>nl = html.find('<STR_LIT:\n>', n, end)<EOL>if nl != -<NUM_LIT:1>:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (nl - n) + '<STR_LIT:\n>'<EOL>n = nl + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (end - n + <NUM_LIT:3>)<EOL>break<EOL><DEDENT><DEDENT>n = end + <NUM_LIT:3><EOL>continue<EOL><DEDENT>is_extended = tag.lower() in extended_tags<EOL><DEDENT>else:<EOL><INDENT>is_extended = False<EOL><DEDENT>while n < len(html):<EOL><INDENT>squote = html.find("<STR_LIT:'>", n)<EOL>dquote = html.find('<STR_LIT:">', n)<EOL>nl = html.find('<STR_LIT:\n>', n)<EOL>angle = html.find('<STR_LIT:>>', n)<EOL>if angle == -<NUM_LIT:1>:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (len(html) - n)<EOL>n = len(html)<EOL>break<EOL><DEDENT>elif -<NUM_LIT:1> < squote < angle or -<NUM_LIT:1> < dquote < angle:<EOL><INDENT>if squote != -<NUM_LIT:1> and dquote != -<NUM_LIT:1>:<EOL><INDENT>if squote < dquote: <EOL><INDENT>open_quote = squote<EOL>quote = "<STR_LIT:'>"<EOL><DEDENT>else:<EOL><INDENT>open_quote = dquote<EOL>quote = '<STR_LIT:">'<EOL><DEDENT><DEDENT>elif dquote != -<NUM_LIT:1>:<EOL><INDENT>open_quote = dquote<EOL>quote = '<STR_LIT:">'<EOL><DEDENT>else:<EOL><INDENT>open_quote = squote<EOL>quote = "<STR_LIT:'>"<EOL><DEDENT>close_quote = html.find(quote, open_quote + <NUM_LIT:1>)<EOL>while n < close_quote:<EOL><INDENT>nl = html.find('<STR_LIT:\n>', n, close_quote)<EOL>if nl == -<NUM_LIT:1>: break<EOL>yield '<STR_LIT:U+0020>' * (nl - n) + '<STR_LIT:\n>'<EOL>n = nl + <NUM_LIT:1><EOL><DEDENT>yield '<STR_LIT:U+0020>' * (close_quote + <NUM_LIT:1> - n)<EOL>n = close_quote + <NUM_LIT:1><EOL>continue<EOL><DEDENT>elif nl == -<NUM_LIT:1> or angle < nl:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (angle + <NUM_LIT:1> - n)<EOL>n = angle + <NUM_LIT:1><EOL>if is_extended and html[angle - <NUM_LIT:1>] != '<STR_LIT:/>':<EOL><INDENT>while n < len(html):<EOL><INDENT>nl = html.find('<STR_LIT:\n>', n)<EOL>close = html.find('<STR_LIT>', n)<EOL>close2 = html.find('<STR_LIT>', close + <NUM_LIT:2>)<EOL>angle = html.find('<STR_LIT:>>', close + <NUM_LIT:2>)<EOL>if nl != -<NUM_LIT:1> and nl < close:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (nl - n) + '<STR_LIT:\n>'<EOL>n = nl + <NUM_LIT:1><EOL><DEDENT>elif close == -<NUM_LIT:1> or angle == -<NUM_LIT:1>:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (len(html) - n)<EOL>n = len(html)<EOL>break<EOL><DEDENT>elif close2 != -<NUM_LIT:1> and close2 < angle:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (close + <NUM_LIT:2> - n)<EOL>n = close + <NUM_LIT:2><EOL><DEDENT>elif html[close + <NUM_LIT:2>:angle].lower() == tag.lower():<EOL><INDENT>yield '<STR_LIT:U+0020>' * (angle + <NUM_LIT:1> - n)<EOL>n = angle + <NUM_LIT:1><EOL>break<EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (angle + <NUM_LIT:1> - n)<EOL>n = angle + <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>break<EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT:U+0020>' * (nl - n) + '<STR_LIT:\n>'<EOL>n = nl + <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>

generator that yields clean visible as it transitions through
    states in the raw `html`


if not isinstance(_html, str):<EOL><INDENT>_html = str(_html, '<STR_LIT:utf-8>')<EOL><DEDENT>non_tag = '<STR_LIT>'.join(non_tag_chars_from_raw(_html))<EOL>return non_tag.encode('<STR_LIT:utf-8>')<EOL>

Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
    returns a Unicode string with all tags replaced by whitespace. In
    particular, all Unicode characters inside HTML are replaced with a
    single whitespace character.

    This *does* detect comments, style, script, link tags and replaces
    them with whitespace.  This is subtle because these tags can be
    self-closing or not.

    It does do anything with HTML-escaped characters.

    Pre-existing whitespace of any kind *except* newlines (\n) and
    linefeeds (\r\n) is converted to single spaces ' ', which has the
    same byte length (and character length).  Newlines and linefeeds
    are left unchanged.

    This is a simple state machine iterator without regexes


_clean = open(clean_visible_path, '<STR_LIT:wb>')<EOL>_clean.write('<STR_LIT>')<EOL>_clean.write('<STR_LIT>')<EOL>for idx, si in enumerate(i_chunk):<EOL><INDENT>if si.stream_id is None:<EOL><INDENT>stream_id = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>stream_id = si.stream_id<EOL><DEDENT>doc = lxml.etree.Element("<STR_LIT>", stream_id=stream_id)<EOL>if si.body and si.body.clean_visible:<EOL><INDENT>try:<EOL><INDENT>doc.text = si.body.clean_visible.decode('<STR_LIT:utf8>')<EOL><DEDENT>except ValueError:<EOL><INDENT>doc.text = drop_invalid_and_upper_utf8_chars(<EOL>si.body.clean_visible.decode('<STR_LIT:utf8>'))<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.critical(traceback.format_exc(exc))<EOL>logger.critical('<STR_LIT>',<EOL>si.stream_id)<EOL>logger.critical(repr(si.body.clean_visible))<EOL>logger.critical('<STR_LIT>', si.stream_id)<EOL>raise<EOL><DEDENT><DEDENT>else:<EOL><INDENT>doc.text = '<STR_LIT>'<EOL><DEDENT>_clean.write(lxml.etree.tostring(doc, encoding='<STR_LIT>'))<EOL><DEDENT>_clean.write('<STR_LIT>')<EOL>_clean.close()<EOL>logger.info(clean_visible_path)<EOL>'''<STR_LIT>'''<EOL>


assert isinstance(span, str),'<STR_LIT>' % span<EOL>span = penn_treebank_brackets.sub('<STR_LIT:U+0020>', span)<EOL>if lower:<EOL><INDENT>span = span.lower()<EOL><DEDENT>span = span.translate(strip_punctuation)<EOL>span = whitespace.sub('<STR_LIT:U+0020>', span)<EOL>return span.strip()<EOL>

Convert a unicode string into a lowercase string with no
punctuation and only spaces for whitespace.

Replace PennTreebank escaped brackets with ' ':
-LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
(The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
http://www.cis.upenn.edu/~treebank/tokenization.html

:param span: string


import argparse<EOL>import sys<EOL>parser = argparse.ArgumentParser()<EOL>parser.add_argument('<STR_LIT:path>')<EOL>args = parser.parse_args()<EOL>html = open(args.path).read()<EOL>html = html.decode('<STR_LIT:utf8>')<EOL>cursor = <NUM_LIT:0><EOL>for s in non_tag_chars_from_raw(html):<EOL><INDENT>for c in s:<EOL><INDENT>if c != '<STR_LIT:U+0020>' and c != html[cursor]:<EOL><INDENT>import pdb; pdb.set_trace()<EOL><DEDENT>sys.stdout.write(c.encode('<STR_LIT:utf8>'))<EOL>sys.stdout.flush()<EOL>cursor += <NUM_LIT:1><EOL><DEDENT><DEDENT>

manual test loop for make_clean_visible_from_raw


def tryload_stage(self, moduleName, functionName, name=None):

if name is None:<EOL><INDENT>name = functionName<EOL><DEDENT>try:<EOL><INDENT>mod = __import__(moduleName, globals(), locals(), [functionName])<EOL><DEDENT>except ImportError as exc:<EOL><INDENT>logger.warn('<STR_LIT>'<EOL>.format(name, moduleName), exc_info=exc)<EOL>return<EOL><DEDENT>if not hasattr(mod, functionName):<EOL><INDENT>logger.warn('<STR_LIT>'<EOL>.format(name, moduleName, functionName))<EOL>return<EOL><DEDENT>self[name] = getattr(mod, functionName)<EOL>

Try to load a stage into self, ignoring errors.

        If loading a module fails because of some subordinate load
        failure, just give a warning and move on.  On success the
        stage is added to the stage dictionary.

        :param str moduleName: name of the Python module
        :param str functionName: name of the stage constructor
        :param str name: name of the stage, defaults to `functionName`


mod = imp.load_source('<STR_LIT>', path)<EOL>self.update(mod.Stages)<EOL>

Add external stages from the Python module in `path`.

        `path` must be a path to a Python module source that contains
        a `Stages` dictionary, which is a map from stage name to callable.

        :param str path: path to the module file


if isinstance(mod, str):<EOL><INDENT>mod = __import__(mod, globals=globals(), locals=locals(),<EOL>fromlist=['<STR_LIT>'], level=<NUM_LIT:0>)<EOL><DEDENT>if not hasattr(mod, '<STR_LIT>'):<EOL><INDENT>raise ImportError(mod)<EOL><DEDENT>self.update(mod.Stages)<EOL>

Add external stages from the Python module `mod`.

        If `mod` is a string, then it will be interpreted as the name
        of a module; otherwise it is an actual module object.  The
        module should exist somewhere in :data:`sys.path`.  The module
        must contain a `Stages` dictionary, which is a map from stage
        name to callable.

        :param mod: name of the module or the module itself
        :raise exceptions.ImportError: if `mod` cannot be loaded or does
          not contain ``Stages``


subconfig = config.get(name, {})<EOL>ctor = self[name]<EOL>return ctor(subconfig)<EOL>

Construct and configure a stage from known stages.

        `name` must be the name of one of the stages in this.  `config`
        is the configuration dictionary of the containing object, and its `name`
        member will be passed into the stage constructor.

        :param str name: name of the stage
        :param dict config: parent object configuration
        :return: callable stage
        :raise exceptions.KeyError: if `name` is not a known stage


raise NotImplementedError('<STR_LIT>')<EOL>

Process a chunk file at a specific path.

        Implementations should work in place, writing results out to the
        same `chunk_path`.  The pipeline guarantees that this stage
        will have a key ``tmp_dir_path`` in its configuration, and this
        can be used to store an intermediate file which is then renamed
        over `chunk_path`.


Do an orderly shutdown of this stage.

        If the stage spawns a child process, kill it, and do any other
        required cleanup.  :meth:`streamcorpus_pipeline.Pipeline._cleanup`
        will call this method on every batch transform stage, regardless
        of whether this is the currently running stage or not.


@abstractmethod<EOL><INDENT>def process_item(self, stream_item, context):<DEDENT>

Process a single :class:`streamcorpus.StreamItem` object.

        `context` is shared across all stages.  It is guaranteed to
        contain ``i_str`` nominally containing the original input file
        name and ``data`` nominally containing the auxiliary data from
        the work item.

        This function may modify `stream_item` in place, construct
        a new item and return it, or return :const:`None` to drop
        the item.

        :param stream_item: stream item to process
        :paramtype stream_item: :class:`streamcorpus.StreamItem`
        :param dict context: additional shared context data
        :return: same or different stream item or :const:`None`


def __call__(self, stream_item, context):

return self.process_item(stream_item, context)<EOL>

Entry point from the pipeline.

        This implementation calls :meth:`process_item`.


pass<EOL>

Do an orderly shutdown of this stage.

        Incremental transforms are generally simple Python code and
        a complicated shutdown is not required.  The pipeline does not
        call this method.


def _create_column_family(self, family, bytes_columns=[], <EOL>key_validation_class=TIME_UUID_TYPE):

sm = SystemManager(random.choice(self.server_list))<EOL>sm.create_column_family(self.namespace, family, super=False,<EOL>key_validation_class = key_validation_class, <EOL>default_validation_class  = TIME_UUID_TYPE,<EOL>column_name_class = ASCII_TYPE)<EOL>for column in bytes_columns:<EOL><INDENT>sm.alter_column(self.namespace, family, column, BYTES_TYPE)<EOL><DEDENT>sm.close()<EOL>

Creates a column family of the name 'family' and sets any of
the names in the bytes_column list to have the BYTES_TYPE.

key_validation_class defaults to TIME_UUID_TYPE and could also
be ASCII_TYPE for md5 hash keys, like we use for 'inbound'


def _create_counter_column_family(self, family, counter_columns=[],<EOL>key_validation_class=UTF8Type):

sm = SystemManager(random.choice(self.server_list))<EOL>sm.create_column_family(self.namespace, family, super=False,<EOL>key_validation_class = key_validation_class, <EOL>default_validation_class="<STR_LIT>",<EOL>column_name_class = ASCII_TYPE)<EOL>for column in counter_columns:<EOL><INDENT>sm.alter_column(self.namespace, family, column, COUNTER_COLUMN_TYPE)<EOL><DEDENT>sm.close()<EOL>


for row in self._tasks.get_range():<EOL><INDENT>logger.debug(row)<EOL>if not row[<NUM_LIT:0>].startswith(key_prefix):<EOL><INDENT>continue<EOL><DEDENT>data = json.loads(row[<NUM_LIT:1>]['<STR_LIT>'])<EOL>data['<STR_LIT>'] = row[<NUM_LIT:0>]<EOL>yield data<EOL><DEDENT>

generate the data objects for every task


c = <NUM_LIT:1><EOL>keeper = None<EOL>for row in self._available.get_range(row_count=max_iter, read_consistency_level=pycassa.ConsistencyLevel.ALL):<EOL><INDENT>logger.debug('<STR_LIT>' % (row,))<EOL>if random.random() < <NUM_LIT:1> / c:<EOL><INDENT>keeper = row[<NUM_LIT:0>]<EOL><DEDENT>if c == max_iter:<EOL><INDENT>break<EOL><DEDENT>c += <NUM_LIT:1><EOL><DEDENT>return keeper<EOL>

get a random key out of the first max_iter rows


<EOL>def _file_type_stats(stream_item, context):<EOL><INDENT>if stream_item.body and stream_item.body.raw:<EOL><INDENT>if '<STR_LIT>' in stream_item.body.raw[:<NUM_LIT>].lower():<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>if has_tags(stream_item.body.raw[:<NUM_LIT>]):<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>xml = xml_ish.search(stream_item.body.raw)<EOL>if xml:<EOL><INDENT>print('<STR_LIT>' % repr(xml.group('<STR_LIT>')))<EOL><DEDENT>else:<EOL><INDENT>pdf = pdf_start.search(stream_item.body.raw)<EOL>if pdf:<EOL><INDENT>print('<STR_LIT>' % repr(pdf.group('<STR_LIT:version>')))<EOL><DEDENT>else:<EOL><INDENT>ext = stream_item.abs_url.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>if len(ext) < <NUM_LIT:6>:<EOL><INDENT>print('<STR_LIT>' % repr(ext))<EOL><DEDENT>else:<EOL><INDENT>first = first_letters.match(stream_item.body.raw)<EOL>if first and False:<EOL><INDENT>print('<STR_LIT>' % repr(first.group('<STR_LIT>')))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' % repr(stream_item.body.raw[:<NUM_LIT:50>]))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return stream_item<EOL><DEDENT>return _file_type_stats<EOL>

returns a kba.pipeline "transform" function that generates file
type stats from the stream_items that it sees.  Currently, these
stats are just the first five non-whitespace characters.


for f_match in filename_re.finditer(text):<EOL><INDENT>yield f_match.group('<STR_LIT>'), f_match.group('<STR_LIT>')<EOL><DEDENT>

Iterate over <FILENAME> XML-like tags and tokenize with nltk


for sentence_id, node in enumerate(self.ner_dom.childNodes):<EOL><INDENT>if node.nodeType == node.TEXT_NODE:<EOL><INDENT>assert only_whitespace.match(node.data), repr(node.data)<EOL>self.byte_idx += len(node.data.encode('<STR_LIT:utf-8>'))<EOL>for line in node.data.splitlines(True):<EOL><INDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>logger.debug('<STR_LIT>' % sentence_id)<EOL>more_sentence_remains = True<EOL>while more_sentence_remains:<EOL><INDENT>sent = Sentence()<EOL>tokens = iter( self.tokens( node ) )<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>tok = next(tokens)<EOL>sent.tokens.append(tok)<EOL><DEDENT>except StopIteration:<EOL><INDENT>yield sent<EOL>more_sentence_remains = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

Iterate over <s> XML-like tags and tokenize with nltk


<EOL>tok_string = self._input_string[start:end].encode('<STR_LIT:utf-8>')<EOL>if only_whitespace.match(tok_string):<EOL><INDENT>return None<EOL><DEDENT>tok = Token()<EOL>tok.token = tok_string<EOL>tok.token_num = self.tok_num<EOL>if '<STR_LIT>' in self.config['<STR_LIT>']:<EOL><INDENT>tok.offsets[OffsetType.BYTES] = Offset(<EOL>type =  OffsetType.BYTES,<EOL>first=self.byte_idx + len(self._input_string[:start].encode('<STR_LIT:utf-8>')),<EOL>length=len(tok_string),<EOL>value=self.config['<STR_LIT>'] and tok_string or None,<EOL>)<EOL><DEDENT>if '<STR_LIT>' in self.config['<STR_LIT>']:<EOL><INDENT>tok.offsets[OffsetType.LINES] = Offset(<EOL>type =  OffsetType.LINES,<EOL>first=self.line_idx,<EOL>length=<NUM_LIT:1>,<EOL>value=self.config['<STR_LIT>'] and tok_string or None,<EOL>)<EOL><DEDENT>self.tok_num += <NUM_LIT:1><EOL>tok.sentence_pos = self.sent_pos<EOL>self.sent_pos += <NUM_LIT:1><EOL>return tok<EOL>

Instantiates a Token from self._input_string[start:end]


<EOL>self.sent_pos = <NUM_LIT:0><EOL>mention_id = <NUM_LIT:0><EOL>while len(sentence_dom.childNodes) > <NUM_LIT:0>:<EOL><INDENT>node = sentence_dom.childNodes.pop(<NUM_LIT:0>)<EOL>if node.nodeType == node.TEXT_NODE:<EOL><INDENT>for line in node.data.splitlines(True):<EOL><INDENT>self._input_string = line<EOL>for start, end in self.word_tokenizer.span_tokenize(line):<EOL><INDENT>tok = self._make_token(start, end)<EOL>if tok:<EOL><INDENT>yield tok<EOL><DEDENT><DEDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT>self.byte_idx += len(line.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert node.nodeName == '<STR_LIT>', node.nodeName<EOL>chain_id = node.attributes.get('<STR_LIT>').value<EOL>entity_type = node.attributes.get('<STR_LIT>').value<EOL>for node in node.childNodes:<EOL><INDENT>assert node.nodeType == node.TEXT_NODE, node.nodeType<EOL>for line in node.data.splitlines(True):<EOL><INDENT>self._input_string = line<EOL>for start, end in self.word_tokenizer.span_tokenize(line):<EOL><INDENT>tok = self._make_token(start, end)<EOL>if tok:<EOL><INDENT>if entity_type in _PRONOUNS:<EOL><INDENT>tok.mention_type = MentionType.PRO<EOL>tok.entity_type = _ENTITY_TYPES[entity_type]<EOL>attr = Attribute(<EOL>attribute_type=AttributeType.PER_GENDER,<EOL>value=str(_PRONOUNS[entity_type])<EOL>)<EOL>self.attributes.append(attr)<EOL><DEDENT>else:<EOL><INDENT>tok.mention_type = MentionType.NAME<EOL>tok.entity_type = _ENTITY_TYPES[entity_type]<EOL><DEDENT>tok.equiv_id = int(chain_id)<EOL>tok.mention_id = mention_id<EOL>yield tok<EOL><DEDENT><DEDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT>self.byte_idx += len(line.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT>mention_id += <NUM_LIT:1><EOL><DEDENT><DEDENT>

Tokenize all the words and preserve NER labels from ENAMEX tags


lp_parser = LingPipeParser(self.config)<EOL>lp_parser.set(ner_dom)<EOL>sentences = list( lp_parser.sentences() )<EOL>return sentences, lp_parser.relations, lp_parser.attributes<EOL>

parse the sentences and tokens out of the XML


def get_name_info(chunk_path, assert_one_date_hour=False, i_str=None,<EOL>chunk_type=Chunk):

assert i_str is not None, '<STR_LIT>'<EOL>name_info = dict()<EOL>if i_str:<EOL><INDENT>name_info['<STR_LIT>'] = i_str<EOL><DEDENT>else:<EOL><INDENT>name_info['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>i_fname = i_str.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>i_fname = i_fname.split('<STR_LIT:.>')[<NUM_LIT:0>]  <EOL>name_info['<STR_LIT>'] = i_fname <EOL>input_md5s = []<EOL>for part in i_fname.split('<STR_LIT:->'):<EOL><INDENT>if len(part) == <NUM_LIT:32> and is_hex_32.match(part):<EOL><INDENT>input_md5s.append(part)<EOL><DEDENT><DEDENT>name_info['<STR_LIT>'] = '<STR_LIT:->'.join(input_md5s)<EOL>ch = chunk_type(path=chunk_path, mode='<STR_LIT:rb>')<EOL>date_hours = set()<EOL>target_names = set()<EOL>doc_ids = set()<EOL>epoch_ticks = None<EOL>count = <NUM_LIT:0><EOL>try:<EOL><INDENT>for si in ch:<EOL><INDENT>if chunk_type is Chunk:<EOL><INDENT>if epoch_ticks is None:<EOL><INDENT>epoch_ticks = si.stream_time.epoch_ticks<EOL><DEDENT>date_hours.add( si.stream_time.zulu_timestamp[:<NUM_LIT>] )<EOL>doc_ids.add( si.doc_id )<EOL>for annotator_id, ratings in list(si.ratings.items()):<EOL><INDENT>for rating in ratings:<EOL><INDENT>target_name = rating.target.target_id.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>target_names.add( target_name )<EOL><DEDENT><DEDENT><DEDENT>count += <NUM_LIT:1><EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>logger.critical('<STR_LIT>', exc_info=True)<EOL><DEDENT>if hasattr(ch, '<STR_LIT>'):<EOL><INDENT>name_info['<STR_LIT>'] = ch.md5_hexdigest<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>data = open(chunk_path).read()<EOL>name_info['<STR_LIT>'] = hashlib.md5(data).hexdigest()<EOL><DEDENT>except Exception as exc:<EOL><INDENT>logger.critical('<STR_LIT>', exc_info=True)<EOL>name_info['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>name_info['<STR_LIT>'] = count<EOL>name_info['<STR_LIT>'] = epoch_ticks<EOL>name_info['<STR_LIT>'] = '<STR_LIT:->'.join( target_names )<EOL>name_info['<STR_LIT>'] = '<STR_LIT:->'.join( [di[:<NUM_LIT:8>] for di in doc_ids] )<EOL>if chunk_type is Chunk:<EOL><INDENT>if assert_one_date_hour:<EOL><INDENT>assert len(date_hours) == <NUM_LIT:1>,'<STR_LIT>' +repr(date_hours)<EOL><DEDENT>if len(date_hours) > <NUM_LIT:0>:<EOL><INDENT>date_hour = list(date_hours)[<NUM_LIT:0>]<EOL>date_hour = date_hour.replace('<STR_LIT:T>', '<STR_LIT:->')<EOL><DEDENT>else:<EOL><INDENT>assert count == <NUM_LIT:0>, (date_hours, count)<EOL>date_hour = None<EOL><DEDENT>name_info['<STR_LIT>'] = date_hour<EOL><DEDENT>else:<EOL><INDENT>name_info['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>name_info['<STR_LIT>'] = '<STR_LIT>' % (random.randint(<NUM_LIT:0>, <NUM_LIT>),)<EOL>name_info['<STR_LIT>'] = datetime.datetime.utcnow().strftime('<STR_LIT>')<EOL>name_info['<STR_LIT>'] = datetime.datetime.utcnow().strftime('<STR_LIT>')<EOL>name_info['<STR_LIT>'] = datetime.datetime.utcnow().strftime('<STR_LIT>')<EOL>return name_info<EOL>

takes a chunk blob and obtains the date_hour, md5, num

makes fields:
i_str
input_fname
input_md5 - parsed from input filename if it contains '-%(md5)s-'
md5
num
epoch_ticks
target_names
doc_ids_8
date_hour
rand8
date_now
time_now
date_time_now


def read_to( idx_bytes, stop_bytes=None, run_bytes=None ):

idx = None<EOL>vals = []<EOL>next_b = None<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>idx, next_b = next(idx_bytes)<EOL><DEDENT>except StopIteration:<EOL><INDENT>idx = None<EOL>next_b = None<EOL>break<EOL><DEDENT>if stop_bytes and next_b in stop_bytes:<EOL><INDENT>break<EOL><DEDENT>if run_bytes and next_b not in run_bytes:<EOL><INDENT>break<EOL><DEDENT>vals.append( next_b )<EOL><DEDENT>return idx, b'<STR_LIT>'.join(vals), next_b<EOL>

iterates through idx_bytes until a byte in stop_bytes or a byte
not in run_bytes.

:rtype (int, string): idx of last byte and all of bytes including
the terminal byte from stop_bytes or not in run_bytes


<EOL>while <NUM_LIT:1>:<EOL><INDENT>idx, attr_name, next_b = read_to(idx_bytes, ['<STR_LIT:=>', '<STR_LIT:>>'])<EOL>attr_vals = []<EOL>if next_b is None or next_b == '<STR_LIT:>>':<EOL><INDENT>return<EOL><DEDENT>idx, space, quote = read_to(idx_bytes, run_bytes = ['<STR_LIT:U+0020>', '<STR_LIT:\t>', '<STR_LIT:\n>', '<STR_LIT:\r>'])<EOL>if quote not in ['<STR_LIT:">', "<STR_LIT:'>"]:<EOL><INDENT>attr_vals = [quote]<EOL>quote = '<STR_LIT:U+0020>'<EOL><DEDENT>idx, attr_val, next_b = read_to(idx_bytes, [quote, '<STR_LIT:>>'])<EOL>assert next_b in [quote, '<STR_LIT:>>', None], attr_val<EOL>attr_vals.append( attr_val )<EOL>yield idx, attr_name.strip(), b'<STR_LIT>'.join(attr_vals).strip()<EOL><DEDENT>

called when idx_chars is just past "<a " inside an HTML anchor tag

generates tuple(end_idx, attr_name, attr_value)


if self.config['<STR_LIT>']:<EOL><INDENT>if not href.lower().startswith(('<STR_LIT>', '<STR_LIT>')):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if self.config['<STR_LIT>']:<EOL><INDENT>return True<EOL><DEDENT>if self.config['<STR_LIT>']:<EOL><INDENT>parts = href.split('<STR_LIT:/>')<EOL>if len(parts) < <NUM_LIT:3>:<EOL><INDENT>return False<EOL><DEDENT>domain = parts[<NUM_LIT:2>].lower()<EOL>for substring in self.config['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>if substring in domain:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>except Exception as exc:<EOL><INDENT>logger.warn('<STR_LIT>', substring, domain, exc_info=True)<EOL>return False<EOL><DEDENT><DEDENT><DEDENT>

Test whether an href string meets criteria specified by
configuration parameters 'require_abs_url', which means "does
it look like it is probably an absolute URL?" and
'domain_substrings'.  It searches for each of the
domain_substrings in the href individually, and if any match,
then returns True.

:param: href string
:returns bool:


idx = <NUM_LIT:0><EOL>new_clean_html = '<STR_LIT>'<EOL>newlines_added = <NUM_LIT:0><EOL>parts = self.clean_html.split('<STR_LIT>')<EOL>assert len('<STR_LIT>'.join(parts) ) == len(self.clean_html)<EOL>for i in range(len(parts)):<EOL><INDENT>part = parts[i]<EOL>m = anchors_re.match(part)<EOL>if not m:<EOL><INDENT>new_clean_html += part<EOL>if i < len(parts) - <NUM_LIT:1>:<EOL><INDENT>new_clean_html += '<STR_LIT>'<EOL><DEDENT>continue<EOL><DEDENT>before = m.group('<STR_LIT>')<EOL>ahref = m.group('<STR_LIT>')<EOL>pre_anchor_increment = before + ahref<EOL>idx += len( pre_anchor_increment.splitlines() )<EOL>first = idx<EOL>length = len(m.group('<STR_LIT>').split('<STR_LIT:\n>'))<EOL>new_clean_html += pre_anchor_increment + '<STR_LIT:\n>' + m.group('<STR_LIT>') + '<STR_LIT>'<EOL>newlines_added += <NUM_LIT:2><EOL>idx += length - <NUM_LIT:1><EOL>yield m.group('<STR_LIT>'), first, length, m.group('<STR_LIT>')<EOL><DEDENT>assert len(self.clean_html) == len(new_clean_html) - newlines_added<EOL>self.clean_html = new_clean_html<EOL>

simple, regex-based extractor of anchor tags, so we can
compute LINE offsets for anchor texts and associate them with
their href.

Generates tuple(href_string, first_byte, byte_length, anchor_text)

Also, this mangles the body.clean_html so that LINE offsets
uniquely identify tokens in anchor tags -- quite a kludge.


input_buffer = self.clean_html<EOL>if chars:<EOL><INDENT>input_buffer = input_buffer.decode('<STR_LIT:utf8>')<EOL><DEDENT>idx = <NUM_LIT:0><EOL>parts = input_buffer.split('<STR_LIT>')<EOL>assert len('<STR_LIT>'.join(parts) ) == len(input_buffer)<EOL>for part in parts:<EOL><INDENT>m = anchors_re.match(part)<EOL>if not m:<EOL><INDENT>idx += len(part) + <NUM_LIT:4><EOL>continue<EOL><DEDENT>before = m.group('<STR_LIT>')<EOL>ahref = m.group('<STR_LIT>')<EOL>idx += len(before) + len(ahref)<EOL>first = idx<EOL>length = len(m.group('<STR_LIT>'))<EOL>idx += length + <NUM_LIT:4><EOL>if chars:<EOL><INDENT>yield m.group('<STR_LIT>').encode('<STR_LIT:utf8>'), first, length, m.group('<STR_LIT>').encode('<STR_LIT:utf8>')<EOL><DEDENT>else:<EOL><INDENT>yield m.group('<STR_LIT>'), first, length, m.group('<STR_LIT>')<EOL><DEDENT><DEDENT>assert idx - <NUM_LIT:4> == len(input_buffer)<EOL>

simple, regex-based extractor of anchor tags, so we can
compute BYTE offsets for anchor texts and associate them with
their href.

Generates tuple(href_string, first_byte, byte_length, anchor_text)


tag_depth = <NUM_LIT:0><EOL>a_tag_depth = <NUM_LIT:0><EOL>vals = []<EOL>href = None<EOL>idx_bytes = enumerate( self.clean_html )<EOL>while <NUM_LIT:1>:<EOL><INDENT>end_idx, val, next_b = read_to( idx_bytes, '<STR_LIT:<>' )<EOL>tag_depth += <NUM_LIT:1><EOL>if href:<EOL><INDENT>assert a_tag_depth > <NUM_LIT:0>, (href, self.clean_html)<EOL>vals.append(val)<EOL><DEDENT>idx, next_b1 = next(idx_bytes)<EOL>if next_b1.lower() == '<STR_LIT:a>':<EOL><INDENT>idx, next_b2 = next(idx_bytes)<EOL>if next_b2 == '<STR_LIT:U+0020>':<EOL><INDENT>a_tag_depth += <NUM_LIT:1><EOL>href = None<EOL>for idx, attr_name, attr_val in iter_attrs( idx_bytes ):<EOL><INDENT>if attr_name.lower() == '<STR_LIT>':<EOL><INDENT>href = attr_val<EOL><DEDENT>if idx is None:<EOL><INDENT>return<EOL><DEDENT><DEDENT>first = idx + <NUM_LIT:1><EOL>if href:<EOL><INDENT>vals = []<EOL><DEDENT><DEDENT><DEDENT>elif next_b1 == '<STR_LIT:/>':<EOL><INDENT>idx, next_b1 = next(idx_bytes)<EOL>if next_b1 == '<STR_LIT:a>':<EOL><INDENT>idx, next_b2 = next(idx_bytes)<EOL>if next_b2 == '<STR_LIT:>>':<EOL><INDENT>a_tag_depth -= <NUM_LIT:1><EOL>if href:<EOL><INDENT>anchor = b'<STR_LIT>'.join(vals)<EOL>length = len(anchor)<EOL>yield href, first, len(anchor), anchor<EOL>href = None<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>vals.append(next_b)<EOL><DEDENT><DEDENT>

byte-based state machine extractor of anchor tags, so we can
compute byte offsets for anchor texts and associate them with
their href.

Generates tuple(href_string, first_byte, byte_length, anchor_text)


def make_labels(self, clean_html, clean_visible=None):

if   self.offset_type == OffsetType.BYTES:<EOL><INDENT>parser = self.byte_href_anchors<EOL><DEDENT>elif self.offset_type == OffsetType.CHARS:<EOL><INDENT>parser = self.char_href_anchors<EOL><DEDENT>elif self.offset_type == OffsetType.LINES:<EOL><INDENT>parser = self.line_href_anchors<EOL><DEDENT>labels = []<EOL>self.clean_html = clean_html<EOL>for href, first, length, value in parser():<EOL><INDENT>if self.href_filter(href):<EOL><INDENT>'''<STR_LIT>'''<EOL>label = Label(<EOL>annotator = Annotator(annotator_id = '<STR_LIT>'),<EOL>target = Target(target_id = href),<EOL>)<EOL>label.offsets[self.offset_type] = Offset(<EOL>first=first, length=length, <EOL>value=value,<EOL>content_form='<STR_LIT>')<EOL>labels.append(label)<EOL><DEDENT><DEDENT>return labels<EOL>

Make a list of Labels for 'author' and the filtered hrefs &
anchors


<EOL>assert self.config.get('<STR_LIT>', True)<EOL>if stream_item.body and stream_item.body.clean_html:<EOL><INDENT>labels = self.make_labels(stream_item.body.clean_html,<EOL>stream_item.body.clean_visible)<EOL>if labels:<EOL><INDENT>if self.offset_type == OffsetType.LINES:<EOL><INDENT>stream_item.body.clean_html = self.clean_html<EOL><DEDENT>stream_item.body.labels['<STR_LIT>'] = []<EOL>add_annotation(stream_item.body, *labels)<EOL><DEDENT><DEDENT>return stream_item<EOL>

Act as an incremental transform in the kba.pipeline


<EOL>tmp_chunk_path = chunk_path + '<STR_LIT:_>'<EOL>t_chunk = Chunk(path=tmp_chunk_path, mode='<STR_LIT:wb>')<EOL>for num, si in enumerate(Chunk(path=chunk_path)):<EOL><INDENT>if num < self.config['<STR_LIT>']:<EOL><INDENT>t_chunk.add(si)<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>t_chunk.close()<EOL>os.rename(tmp_chunk_path, chunk_path)<EOL>

batch-type transform stage: reads a chunk from chunk_path, and
replaces it with a new chunk at the same path


<EOL>fake_equiv_ids = -<NUM_LIT:2><EOL>equiv_ids = collections.defaultdict(lambda: (set(), set()))<EOL>for tagger_id, sents in list(sentences.items()):<EOL><INDENT>for sent in sents:<EOL><INDENT>for tok in sent.tokens:<EOL><INDENT>if tok.entity_type is not None:<EOL><INDENT>if tok.equiv_id == -<NUM_LIT:1>:<EOL><INDENT>eqid = fake_equiv_ids<EOL>fake_equiv_ids -= <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>eqid = tok.equiv_id<EOL><DEDENT>equiv_ids[eqid][<NUM_LIT:0>].add(cleanse(tok.token.decode('<STR_LIT:utf8>')))<EOL>equiv_ids[eqid][<NUM_LIT:1>].add(tok)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return equiv_ids<EOL>

assemble in-doc coref chains by mapping equiv_id to tokens and
their cleansed name strings

:param sentences: iterator over token generators
:returns dict:
    keys are equiv_ids,
    values are tuple(concatentated name string, list of tokens)


found_all = True<EOL>for name in target_mentions:<EOL><INDENT>found_one = False<EOL>for chain_ment in chain_mentions:<EOL><INDENT>if name in chain_ment:<EOL><INDENT>found_one = True<EOL>break<EOL><DEDENT><DEDENT>if not found_one:<EOL><INDENT>found_all = False<EOL>break<EOL><DEDENT><DEDENT>return found_all<EOL>

For each name string in the target_mentions list, searches through
all chain_mentions looking for any cleansed Token.token that
contains the name.  Returns True only if all of the target_mention
strings appeared as substrings of at least one cleansed
Token.token.  Otherwise, returns False.

:type target_mentions: list of basestring
:type chain_mentions: list of basestring

:returns bool:


for multi_token_name in multi_token_target_mentions:<EOL><INDENT>if ALL_mentions(multi_token_name.split(), chain_mentions):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

For each name string (potentially consisting of multiple tokens) in the
target_mentions list, searches through all chain_mentions looking for any
cleansed Token.token that contains all the tokens in the name.  Returns
True only if all of the target_mention strings appeared as substrings of at
least one cleansed Token.token.  Otherwise, returns False.

:type target_mentions: list of basestring
:type chain_mentions: list of basestring

:returns bool:


for name in target_mentions:<EOL><INDENT>for chain_ment in chain_mentions:<EOL><INDENT>if name in chain_ment:<EOL><INDENT>return True<EOL><DEDENT><DEDENT><DEDENT>return False<EOL>

For each name string in the target_mentions list, searches through
all chain_mentions looking for any cleansed Token.token that
contains the name.  Returns True if any of the target_mention
strings appeared as substrings of any cleansed Token.token.
Otherwise, returns False.

:type target_mentions: list of basestring
:type chain_mentions: list of basestring

:returns bool:


chain_selector = aligner_data.get('<STR_LIT>', '<STR_LIT>')<EOL>assert chain_selector in _CHAIN_SELECTORS,'<STR_LIT>' % (chain_selector, list(_CHAIN_SELECTORS.keys()))<EOL>chain_selector = _CHAIN_SELECTORS[chain_selector]<EOL>equiv_ids = make_chains_with_names( stream_item.body.sentences )<EOL>required_annotator_id = aligner_data.get('<STR_LIT>')<EOL>for annotator_id, ratings in list(stream_item.ratings.items()):<EOL><INDENT>if (required_annotator_id is not None) and (annotator_id != required_annotator_id):<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>for rating in ratings:<EOL><INDENT>label = Label(annotator=rating.annotator,<EOL>target=rating.target)<EOL>for eqid, (chain_mentions, chain_tokens) in list(equiv_ids.items()):<EOL><INDENT>if chain_selector(rating.mentions, chain_mentions):<EOL><INDENT>for tok in chain_tokens:<EOL><INDENT>add_annotation(tok, label)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

Convert doc-level Rating object into a Label, and add that Label
to all Token in all coref chains identified by
aligner_data["chain_selector"]

:param stream_item: document that has a doc-level Rating to translate into token-level Labels.
:param aligner_data: dict containing:
  chain_selector: ALL or ANY
  annotator_id: string to find at stream_item.Ratings[i].annotator.annotator_id

If chain_selector==ALL, then only apply Label to chains in which
all of the Rating.mentions strings appear as substrings within at
least one of the Token.token strings.

If chain_selector==ANY, then apply Label to chains in which any of
the Rating.mentions strings appear as a substring within at least
one of the Token.token strings.

If chain_selector==ANY_MULTI_TOKEN, then apply Label to chains in which all
the names in any of the Rating.mentions strings appear as a substring within at least
one of the Token.token strings.


<EOL>all_mregexes = []<EOL>for m in rating.mentions:<EOL><INDENT>mregexes = []<EOL>mpatterns = m.decode('<STR_LIT:utf8>').split('<STR_LIT:U+0020>')<EOL>for mpat in mpatterns:<EOL><INDENT>if mpat.startswith('<STR_LIT>') and mpat.endswith('<STR_LIT>'): <EOL><INDENT>mpat = mpat[<NUM_LIT:4>:-<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>mpat = cleanse(mpat)<EOL><DEDENT>if mpat:<EOL><INDENT>mpat = r'<STR_LIT>' % mpat<EOL>logger.debug('<STR_LIT>', mpat)<EOL>mregexes.append(re.compile(mpat, re.UNICODE | re.IGNORECASE))<EOL><DEDENT><DEDENT>if not mregexes:<EOL><INDENT>logger.warn('<STR_LIT>' % (m, rating))<EOL><DEDENT>all_mregexes.append(mregexes)<EOL><DEDENT>for i in range(len(tokens)):<EOL><INDENT>for mregexes in all_mregexes:<EOL><INDENT>if mregexes[<NUM_LIT:0>].match(tokens[i][<NUM_LIT:0>][<NUM_LIT:0>]):<EOL><INDENT>m_j = <NUM_LIT:1><EOL>i_j = <NUM_LIT:0><EOL>last_token_matched = <NUM_LIT:0><EOL>matched = True<EOL>while m_j < len(mregexes):<EOL><INDENT>i_j += <NUM_LIT:1><EOL>if i_j == len(tokens[i + last_token_matched][<NUM_LIT:0>]):<EOL><INDENT>i_j = <NUM_LIT:0><EOL>last_token_matched += <NUM_LIT:1><EOL>if i + last_token_matched == len(tokens):<EOL><INDENT>matched = False<EOL>break<EOL><DEDENT><DEDENT>target_token = tokens[i + last_token_matched][<NUM_LIT:0>][i_j]<EOL>if mregexes[m_j].match(target_token):<EOL><INDENT>m_j += <NUM_LIT:1><EOL><DEDENT>elif target_token == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>matched = False<EOL>break<EOL><DEDENT><DEDENT>if matched:<EOL><INDENT>toks = set()<EOL>for j in range(last_token_matched + <NUM_LIT:1>):<EOL><INDENT>toks.add(tokens[i + j][<NUM_LIT:1>])<EOL><DEDENT>for tok in toks:<EOL><INDENT>yield tok<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

iterate through all tokens looking for matches of cleansed tokens
    or token regexes, skipping tokens left empty by cleansing and
    coping with Token objects that produce multiple space-separated
    strings when cleansed.  Yields tokens that match.


tagger_id = _get_tagger_id(stream_item, aligner_data)<EOL>sentences = stream_item.body.sentences.get(tagger_id)<EOL>if not sentences:<EOL><INDENT>return<EOL><DEDENT>tokens = [(cleanse(tok.token.decode('<STR_LIT:utf8>')).split('<STR_LIT:U+0020>'), tok) for tok in itertools.chain(*[sent.tokens for sent in sentences])]    <EOL>required_annotator_id = aligner_data['<STR_LIT>']<EOL>for annotator_id, ratings in list(stream_item.ratings.items()):<EOL><INDENT>if (required_annotator_id is None) or (annotator_id == required_annotator_id):<EOL><INDENT>for rating in ratings:<EOL><INDENT>label = Label(annotator=rating.annotator,<EOL>target=rating.target)<EOL>num_tokens_matched = <NUM_LIT:0><EOL>for tok in look_ahead_match(rating, tokens):<EOL><INDENT>if aligner_data.get('<STR_LIT>'):<EOL><INDENT>tok.labels.pop(annotator_id, None)<EOL><DEDENT>add_annotation(tok, label)<EOL>num_tokens_matched += <NUM_LIT:1><EOL><DEDENT>if num_tokens_matched == <NUM_LIT:0>:<EOL><INDENT>logger.warning('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>rating.target.target_id,<EOL>stream_item.stream_id)<EOL><DEDENT>else:<EOL><INDENT>logger.debug('<STR_LIT>',<EOL>num_tokens_matched, rating.target.target_id,<EOL>stream_item.stream_id)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

iterate through tokens looking for near-exact matches to strings
in si.ratings...mentions


def make_ner_file(self, clean_visible_path, ner_xml_path):

if self.template is None:<EOL><INDENT>raise exceptions.NotImplementedError(

run tagger a child process to get XML output


def align_chunk_with_ner(self, ner_xml_path, i_chunk, o_chunk):

<EOL>input_iter = i_chunk.__iter__()<EOL>all_ner = xml.dom.minidom.parse(open(ner_xml_path))<EOL>for ner_dom in all_ner.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>stream_item = next(input_iter)<EOL>stream_id = ner_dom.attributes.get('<STR_LIT>').value<EOL>if stream_item.stream_id is None:<EOL><INDENT>assert not stream_id, '<STR_LIT>' % stream_id<EOL>logger.critical('<STR_LIT>')<EOL>continue<EOL><DEDENT>assert stream_id and stream_id == stream_item.stream_id,'<STR_LIT>' % (stream_id, stream_item.stream_id)<EOL>if not stream_item.body:<EOL><INDENT>continue<EOL><DEDENT>tagging = Tagging()<EOL>tagging.tagger_id = self.tagger_id  <EOL>'''<STR_LIT>'''<EOL>tagging.generation_time = streamcorpus.make_stream_time()<EOL>stream_item.body.taggings[self.tagger_id] = tagging       <EOL>sentences, relations, attributes = self.get_sentences(ner_dom)<EOL>stream_item.body.sentences[self.tagger_id] = sentences    <EOL>stream_item.body.relations[self.tagger_id] = relations    <EOL>stream_item.body.attributes[self.tagger_id] = attributes  <EOL>logger.debug('<STR_LIT>' % stream_item.stream_id)<EOL>'''<STR_LIT>'''<EOL>if '<STR_LIT>' in self.config and self.config['<STR_LIT>']:<EOL><INDENT>assert '<STR_LIT>' in self.config, '<STR_LIT>'<EOL>aligner = AlignmentStrategies[ self.config['<STR_LIT>'] ]<EOL>aligner( stream_item, self.config['<STR_LIT>'] )<EOL><DEDENT>gc.collect()<EOL>try:<EOL><INDENT>o_chunk.add(stream_item)<EOL><DEDENT>except MemoryError as exc:<EOL><INDENT>msg = traceback.format_exc(exc)<EOL>msg += make_memory_info_msg()<EOL>logger.critical(msg)<EOL>raise PipelineOutOfMemory(msg)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>o_chunk.close()<EOL>logger.info('<STR_LIT>' % ner_xml_path)<EOL><DEDENT>except MemoryError as exc:<EOL><INDENT>msg = traceback.format_exc(exc)<EOL>msg += make_memory_info_msg()<EOL>logger.critical(msg)<EOL>raise PipelineOutOfMemory(msg)<EOL><DEDENT>

iterate through ner_xml_path to fuse with i_chunk into o_chunk


raise exceptions.NotImplementedError<EOL>

parse the sentences and tokens out of the XML


if self._child:<EOL><INDENT>try:<EOL><INDENT>self._child.terminate()<EOL><DEDENT>except OSError as exc:<EOL><INDENT>if exc.errno == <NUM_LIT:3>:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>

send SIGTERM to the tagger child process


if not '<STR_LIT>' in config:<EOL><INDENT>logger.critical('<STR_LIT>', config)<EOL>raise ConfigurationError('<STR_LIT>')<EOL><DEDENT>root_path = config['<STR_LIT>'].pop('<STR_LIT>', None)<EOL>if not root_path:<EOL><INDENT>root_path = os.getcwd()<EOL><DEDENT>if not root_path.startswith('<STR_LIT:/>'):<EOL><INDENT>root_path = os.path.join( os.getcwd(), root_path )<EOL><DEDENT>def recursive_abs_path( sub_config, root_path ):<EOL><INDENT>for key, val in sub_config.items():<EOL><INDENT>if isinstance(val, basestring):<EOL><INDENT>if key.endswith('<STR_LIT:path>'):<EOL><INDENT>if re.match('<STR_LIT>', val): continue<EOL>if not val.startswith('<STR_LIT:/>'):<EOL><INDENT>sub_config[key] = os.path.join(root_path, val)<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(val, dict):<EOL><INDENT>recursive_abs_path( val, root_path )<EOL><DEDENT><DEDENT><DEDENT>recursive_abs_path( config, root_path )<EOL>config['<STR_LIT>'] = root_path<EOL>

given a config dict with streamcorpus_pipeline as a key, find all
    keys under streamcorpus_pipeline that end with "_path" and if the
    value of that key is a relative path, convert it to an absolute
    path using the value provided by root_path