Looks for presence of template files with priority given to HTTP method ( verb ) and role . Full filenames are <handler > . <role > . <verb > . <ext > where <handler > = lower - case handler name <role > = role of current user <verb > = HTTP verb e . g . GET or POST <ext > = html xml etc . Only <handler > and <ext > are required . Properties module_name and handler_name can be passed in params to override the current module / handler name . Returns : Tuple with first element = template file name and second element = template directory path tuple

def get_view_file(handler, params={}):
    """
    Looks for presence of template files with priority given to
     HTTP method (verb) and role.
    Full filenames are <handler>.<role>.<verb>.<ext> where
     <handler> = lower-case handler name
     <role> = role of current user
     <verb> = HTTP verb, e.g. GET or POST
     <ext> = html, xml, etc.
    Only <handler> and <ext> are required.
    Properties 'module_name' and 'handler_name' can be passed in
     params to override the current module/handler name.

    Returns:
      Tuple with first element = template file name and
      second element = template directory path tuple
    """
    if 'ext' in params:
        desired_ext = params['ext']
    else:
        desired_ext = 'html'

    verb = handler.request.method.lower()
    app_name = ''
    module_name = None
    handler_name = None
    cls = handler.__class__
    if (cls.__module__.startswith('handlers.')
        and cls.__name__.endswith('Handler')):
        handler_path = cls.__module__.split('.')
        if len(handler_path) == 3:
            app_name = to_filename(handler_path[1])
        module_name = to_filename(handler_path[-1])
        handler_name = to_filename(cls.__name__.partition('Handler')[0])

    if 'app_name' in params:
        app_name = params['app_name']
    if 'module_name' in params:
        module_name = params['module_name']
    if 'handler_name' in params:
        handler_name = params['handler_name']

    # Get template directory hierarchy -- Needed if we inherit from templates
    # in directories above us (due to sharing with other templates).
    themes = config.BLOG['theme']
    if isinstance(themes, basestring):
      themes = [themes]
    template_dirs = []
    views_dir = os.path.join(config.APP_ROOT_DIR, 'views')
    for theme in themes:
      root_folder = os.path.join(views_dir, theme)
      if module_name:
          template_dirs += (os.path.join(root_folder, app_name, module_name),)
      if app_name:
          template_dirs += (os.path.join(root_folder, app_name),)
      template_dirs += (root_folder,)

    # Now check possible extensions for the given template file.
    if module_name and handler_name:
        entries = templates.get(app_name, {}).get(module_name, {})
        possible_roles = []
        if users.is_current_user_admin():
            possible_roles.append('.admin.')
        if users.get_current_user():
            possible_roles.append('.user.')
        possible_roles.append('.')
        for role in possible_roles:
            filename = ''.join([handler_name, role, verb, '.', desired_ext])
            if filename in entries:
                return {'file': filename, 'dirs': template_dirs}
        for role in possible_roles:
            filename = ''.join([handler_name, role, desired_ext])
            if filename in entries:
                return {'file': filename, 'dirs': template_dirs}
    return {'file': 'notfound.html', 'dirs': template_dirs}


def __init__(self, cache_time=None):
        """Each ViewPage has a variable cache timeout"""
        if cache_time == None:
            self.cache_time = config.BLOG['cache_time']
        else:
            self.cache_time = cache_time

def __init__ ( self , cache_time = None ) : if cache_time == None : self . cache_time = config . BLOG [ 'cache_time' ] else : self . cache_time = cache_time


def full_render(self, handler, template_info, more_params):
        """Render a dynamic page from scatch."""
        logging.debug("Doing full render using template_file: %s", template_info['file'])
        url = handler.request.uri
        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)

        global NUM_FULL_RENDERS
        if not path in NUM_FULL_RENDERS:
            NUM_FULL_RENDERS[path] = 0
        NUM_FULL_RENDERS[path] += 1     # This lets us see % of cached views
                                        # in /admin/timings (see timings.py)
        tags = Tag.list()

        # Define some parameters it'd be nice to have in views by default.
        template_params = {
            "current_url": url,
            "bloog_version": config.BLOG['bloog_version'],
            "user": users.get_current_user(),
            "user_is_admin": users.is_current_user_admin(),
            "login_url": users.create_login_url(handler.request.uri),
            "logout_url": users.create_logout_url(handler.request.uri),
            "blog": config.BLOG,
            "blog_tags": tags
        }
        template_params.update(config.PAGE)
        template_params.update(more_params)
        return template.render(template_info['file'], template_params,
                               debug=config.DEBUG,
                               template_dirs=template_info['dirs'])


Checks if there s a non - stale cached version of this view and if so return it .

def render_or_get_cache(self, handler, template_info, template_params={}):
        """Checks if there's a non-stale cached version of this view,
           and if so, return it."""
        user = users.get_current_user()
        key = handler.request.url
        if self.cache_time and not user:
            # See if there's a cache within time.
            # The cache key suggests a problem with the url <-> function
            #  mapping, because a significant advantage of RESTful design
            #  is that a distinct url gets you a distinct, cacheable
            #  resource.  If we have to include states like "user?" and
            #  "admin?", then it suggests these flags should be in url.
            # TODO - Think about the above with respect to caching.
            try:
                data = memcache.get(key)
            except ValueError:
                data = None
            if data is not None:
                return data

        output = self.full_render(handler, template_info, template_params)
        if self.cache_time and not user:
            memcache.add(key, output, self.cache_time)
        return output


Can pass overriding parameters within dict . These parameters can include : ext : xml ( or any other format type )

def render(self, handler, params={}):
        """
        Can pass overriding parameters within dict.  These parameters can
        include:
            'ext': 'xml' (or any other format type)
        """
        template_info = get_view_file(handler, params)
        logging.debug("Using template at %s", template_info['file'])
        output = self.render_or_get_cache(handler, template_info, params)
        handler.response.out.write(output)

def
render
(
self
,
handler
,
params
=
{
}
)
:
template_info
=
get_view_file
(
handler
,
params
)
logging
.
debug
(
"Using template at %s"
,
template_info
[
'file'
]
)
output
=
self
.
render_or_get_cache
(
handler
,
template_info
,
params
)
handler
.
response
.
out
.
write
(
output
)


Handles typical rendering of queries into datastore with paging .

def render_query(self, handler, model_name, query, params={},
                     num_limit=config.PAGE['articles_per_page'],
                     num_offset=0):
        """
        Handles typical rendering of queries into datastore
        with paging.
        """
        limit = string.atoi(handler.request.get("limit") or str(num_limit))
        offset = string.atoi(handler.request.get("offset") or str(num_offset))
        # Trick is to ask for one more than you need to see if 'next' needed.
        models = query.fetch(limit+1, offset)
        render_params = {model_name: models, 'limit': limit}
        if len(models) > limit:
            render_params.update({ 'next_offset': str(offset+limit) })
            models.pop()
        if offset > 0:
            render_params.update({ 'prev_offset': str(offset-limit) })
        render_params.update(params)

        self.render(handler, render_params)


A decorator to enforce user roles currently user ( logged in ) and admin .

def role(role):
    """
    A decorator to enforce user roles, currently 'user' (logged in)
    and 'admin'.

    To use it, decorate your handler methods like this:

    import authorized
    @authorized.role("admin")
    def get(self):
      user = users.GetCurrentUser(self)
      self.response.out.write('Hello, ' + user.nickname())

    If this decorator is applied to a GET handler, we check if the
    user is logged in and redirect her to the create_login_url() if not.

    For HTTP verbs other than GET, we cannot do redirects to the login
    url because the return redirects are done as GETs (not the original
    HTTP verb for the handler).  So if the user is not logged in, we
    return an error.
    """
    def wrapper(handler_method):
        def check_login(self, *args, **kwargs):
            user = users.get_current_user()
            if not user:
                if self.request.method != 'GET':
                    logging.debug("Not user - aborting")
                    self.error(403)
                else:
                    logging.debug("User not logged in -- force login")
                    self.redirect(users.create_login_url(self.request.uri))
            elif role == "user" or (role == "admin" and
                                    users.is_current_user_admin()):
                logging.debug("Role is %s so will allow handler", role)
                handler_method(self, *args, **kwargs)
            else:
                if self.request.method == 'GET':
                    logging.debug("Unknown role (%s) on GET", role)
                    self.redirect("/403.html")
                else:
                    logging.debug("Unknown role: %s", role)
                    self.error(403) # User didn't meet role.
                                    # TODO: Give better feedback/status code.
        return check_login
    return wrapper


Renders the template at the given path with the given dict of values .

def render(template_path, template_dict, debug=False, template_dirs=()):
  """Renders the template at the given path with the given dict of values.

  Example usage:
    render("templates/index.html", {"name": "Bret", "values": [1, 2, 3]})

  Args:
    template_path: path to a Django template
    template_dict: dictionary of values to apply to the template
  """
  t = load(template_path, debug, template_dirs)
  return t.render(Context(template_dict))

def render ( template_path , template_dict , debug = False , template_dirs = ( ) ) : t = load ( template_path , debug , template_dirs ) return t . render ( Context ( template_dict ) )


Loads the Django template from the given path .

def load(path, debug=False, template_dirs=()):
  """Loads the Django template from the given path.

  It is better to use this function than to construct a Template using the
  class below because Django requires you to load the template with a method
  if you want imports and extends to work in the template.
  """
  abspath = os.path.abspath(path)

  if not debug:
    template = template_cache.get(abspath, None)
  else:
    template = None

  if not template:
    directory, file_name = os.path.split(abspath)
    if directory:
      template_dirs = [directory] + template_dirs
    new_settings = {
        'TEMPLATE_DIRS': template_dirs,
        'TEMPLATE_DEBUG': debug,
        'DEBUG': debug,
    }
    old_settings = _swap_settings(new_settings)
    try:
      template = django.template.loader.get_template(file_name)
    finally:
      _swap_settings(old_settings)

    if not debug:
      template_cache[abspath] = template

    def wrap_render(context, orig_render=template.render):
      URLNode = django.template.defaulttags.URLNode
      save_urlnode_render = URLNode.render
      old_settings = _swap_settings(new_settings)
      try:
        URLNode.render = _urlnode_render_replacement
        return orig_render(context)
      finally:
        _swap_settings(old_settings)
        URLNode.render = save_urlnode_render

    template.render = wrap_render

  return template

def load ( path , debug = False , template_dirs = ( ) ) : abspath = os . path . abspath ( path ) if not debug : template = template_cache . get ( abspath , None ) else : template = None if not template : directory , file_name = os . path . split ( abspath ) if directory : template_dirs = [ directory ] + template_dirs new_settings = { 'TEMPLATE_DIRS' : template_dirs , 'TEMPLATE_DEBUG' : debug , 'DEBUG' : debug , } old_settings = _swap_settings ( new_settings ) try : template = django . template . loader . get_template ( file_name ) finally : _swap_settings ( old_settings ) if not debug : template_cache [ abspath ] = template def wrap_render ( context , orig_render = template . render ) : URLNode = django . template . defaulttags . URLNode save_urlnode_render = URLNode . render old_settings = _swap_settings ( new_settings ) try : URLNode . render = _urlnode_render_replacement return orig_render ( context ) finally : _swap_settings ( old_settings ) URLNode . render = save_urlnode_render template . render = wrap_render return template


Swap in selected Django settings returning old settings .

def _swap_settings(new):
  """Swap in selected Django settings, returning old settings.

  Example:
    save = _swap_settings({'X': 1, 'Y': 2})
    try:
      ...new settings for X and Y are in effect here...
    finally:
      _swap_settings(save)

  Args:
    new: A dict containing settings to change; the keys should
      be setting names and the values settings values.

  Returns:
    Another dict structured the same was as the argument containing
    the original settings.  Original settings that were not set at all
    are returned as None, and will be restored as None by the
    'finally' clause in the example above.  This shouldn't matter; we
    can't delete settings that are given as None, since None is also a
    legitimate value for some settings.  Creating a separate flag value
    for 'unset' settings seems overkill as there is no known use case.
  """
  settings = django.conf.settings
  old = {}
  for key, value in new.iteritems():
    old[key] = getattr(settings, key, None)
    setattr(settings, key, value)
  return old

def _swap_settings ( new ) : settings = django . conf . settings old = { } for key , value in new . iteritems ( ) : old [ key ] = getattr ( settings , key , None ) setattr ( settings , key , value ) return old


Used to extend the Django template library with custom filters and tags .

def create_template_register():
  """Used to extend the Django template library with custom filters and tags.

  To extend the template library with a custom filter module, create a Python
  module, and create a module-level variable named "register", and register
  all custom filters to it as described at
  http://www.djangoproject.com/documentation/templates_python/
    #extending-the-template-system:

    templatefilters.py
    ==================
    register = webapp.template.create_template_register()

    def cut(value, arg):
      return value.replace(arg, '')
    register.filter(cut)

  Then, register the custom template module with the register_template_module
  function below in your application module:

    myapp.py
    ========
    webapp.template.register_template_module('templatefilters')
  """
  return django.template.Library()

def create_template_register ( ) : return django . template . Library ( )


Registers a template extension module to make it usable in templates .

def register_template_library(package_name):
  """Registers a template extension module to make it usable in templates.

  See the documentation for create_template_register for more information."""
  if not django.template.libraries.get(package_name, None):
    django.template.add_to_builtins(package_name)

def register_template_library ( package_name ) : if not django . template . libraries . get ( package_name , None ) : django . template . add_to_builtins ( package_name )


Replacement for django s { % url % } block .

def _urlnode_render_replacement(self, context):
  """Replacement for django's {% url %} block.

  This version uses WSGIApplication's url mapping to create urls.

  Examples:

  <a href="{% url MyPageHandler "overview" %}">
  {% url MyPageHandler implicit_args=False %}
  {% url MyPageHandler "calendar" %}
  {% url MyPageHandler "jsmith","calendar" %}
  """
  args = [arg.resolve(context) for arg in self.args]
  try:
    app = webapp.WSGIApplication.active_instance
    handler = app.get_registered_handler_by_name(self.view_name)
    return handler.get_url(implicit_args=True, *args)
  except webapp.NoUrlFoundError:
    return ''

def _urlnode_render_replacement ( self , context ) : args = [ arg . resolve ( context ) for arg in self . args ] try : app = webapp . WSGIApplication . active_instance handler = app . get_registered_handler_by_name ( self . view_name ) return handler . get_url ( implicit_args = True , * args ) except webapp . NoUrlFoundError : return ''


Processes HTML for embedded code using SyntaxHighlighter Determines languages used by checking class attribute of pre tags with name = code .

def process_html(html):
    """Processes HTML for embedded code using SyntaxHighlighter

    Determines languages used by checking class attribute of pre tags
    with name="code".

    Args:
      html: HTML to be processed for embedded code

    Returns:
      The modified html and a list of strings giving the embedded
      code languages.
    """
    code_tag = re.compile('\s*
',
                          re.MULTILINE)
    languages = set([])
    soup = BeautifulSoup(html)
    clean_html = ''
    for section in soup.contents:
        txt = str(section)
        matchobj = re.match(code_tag, txt)
        if matchobj:
            languages.add(matchobj.group(1))
            clean_html += re.sub(r'
', "\n", txt)
        else:
            clean_html += txt

    # Map the language class names to the spelling for javascript files
    list_language_files = [language_jsfiles[lang] for lang in list(languages)]
    return clean_html.decode('utf-8'), list_language_files


Parses HTML and tries to sanitize it using white list . This method is a mishmash of code from Django snippets ( http : // www . djangosnippets . org / snippets / 169 ) and the HTML sanitization of Universal Feed Parser . It explicitly looks for valid hrefs to prevent scripts lurking in there . Unfortunately style either as a tag or attribute can contain malicious script through executable CSS definitions . So sanitized HTML cannot be colored or highlighted using styles .

def sanitize_html(html='
No comment
', encoding=None,
                  allow_tags=[], allow_attributes=[],
                  blacklist_tags=[], blacklist_attributes=[],
                  trusted_source=False):
    """Parses HTML and tries to sanitize it using white list.

    This method is a mishmash of code from Django snippets
    (http://www.djangosnippets.org/snippets/169) and the
    HTML sanitization of Universal Feed Parser.  It explicitly
    looks for valid hrefs to prevent scripts lurking in there.
    Unfortunately, style, either as a tag or attribute, can
    contain malicious script through executable CSS definitions.
    So sanitized HTML cannot be colored or highlighted using styles.

    Args:
      html: HTML to be sanitized.
      allow_tags: limit all tags to just this list
      allow_attributes: limit all tags to just this list
      allow_styling: should only be TRUE if you trust source

    Returns:
      Sanitized version of html

    Raises:
      DangerousHTMLError if the supplied HTML has dangerous elements.
    """
    if not allow_tags:
        allow_tags = acceptable_tags
    if not allow_attributes:
        allow_attributes = acceptable_attributes
    allow_tags = [tag for tag in allow_tags if tag not in blacklist_tags]
    allow_attributes = [tag for tag in allow_attributes
                        if tag not in blacklist_tags]
    if trusted_source:
        allow_attributes += attributes_for_trusted_source
        allow_tags += tags_for_trusted_source

    if isinstance(html, unicode) and not encoding:
        logging.debug("Sanitizing unicode input.")
        soup = BeautifulSoup(html,
                            convertEntities=BeautifulSoup.XHTML_ENTITIES)
    else:
        if not encoding:
            encoding = 'latin-1'
        logging.debug("Sanitizing string input, assuming %s", encoding)
        soup = BeautifulSoup(html.decode(encoding, 'ignore'),
                             convertEntities=BeautifulSoup.XHTML_ENTITIES)
    for comment in soup.findAll(
                    text = lambda text: isinstance(text, Comment)):
        comment.extract()
    for tag in soup.findAll(True):
        if tag.name not in allow_tags:
            tag.hidden = True
            if tag.name in danger_elements:
                raise DangerousHTMLError(html)
        ok_attrs = []
        for attr, val in tag.attrs:
            if attr == 'href' and not href_matcher.match(val) and not trusted_source:
                continue
            if attr in allow_attributes:
                if attr in js_possible_attributes:
                    if javascript_matcher.match(val):
                        raise DangerousHTMLError(html)
                ok_attrs += [(attr, val)]
        tag.attrs = ok_attrs
    return soup.renderContents().decode('utf-8')

def
sanitize_html
(
html
=
'
No comment
'
,
encoding
=
,
allow_tags
=
[
]
,
allow_attributes
=
[
]
,
blacklist_tags
=
[
]
,
blacklist_attributes
=
[
]
,
trusted_source
=
False
)
:
if
not
allow_tags
:
allow_tags
=
acceptable_tags
if
not
allow_attributes
:
allow_attributes
=
acceptable_attributes
allow_tags
=
[
tag
for
tag
in
allow_tags
if
tag
not
in
blacklist_tags
]
allow_attributes
=
[
tag
for
tag
in
allow_attributes
if
tag
not
in
blacklist_tags
]
if
trusted_source
:
allow_attributes
+=
attributes_for_trusted_source
allow_tags
+=
tags_for_trusted_source
if
isinstance
(
html
,
unicode
)
and
not
encoding
:
logging
.
debug
(
"Sanitizing unicode input."
)
soup
=
BeautifulSoup
(
html
,
convertEntities
=
BeautifulSoup
.
XHTML_ENTITIES
)
else
:
if
not
encoding
:
encoding
=
'latin-1'
logging
.
debug
(
"Sanitizing string input, assuming %s"
,
encoding
)
soup
=
BeautifulSoup
(
html
.
decode
(
encoding
,
'ignore'
)
,
convertEntities
=
BeautifulSoup
.
XHTML_ENTITIES
)
for
comment
in
soup
.
findAll
(
text
=
lambda
text
:
isinstance
(
text
,
Comment
)
)
:
comment
.
extract
(
)
for
tag
in
soup
.
findAll
(
True
)
:
if
tag
.
name
not
in
allow_tags
:
tag
.
hidden
=
True
if
tag
.
name
in
danger_elements
:
raise
DangerousHTMLError
(
html
)
ok_attrs
=
[
]
for
attr
,
val
in
tag
.
attrs
:
if
attr
==
'href'
and
not
href_matcher
.
match
(
val
)
and
not
trusted_source
:
continue
if
attr
in
allow_attributes
:
if
attr
in
js_possible_attributes
:
if
javascript_matcher
.
match
(
val
)
:
raise
DangerousHTMLError
(
html
)
ok_attrs
+=
[
(
attr
,
val
)
]
tag
.
attrs
=
ok_attrs
return
soup
.
renderContents
(
)
.
decode
(
'utf-8'
)


def chop_up(text, chop_size=5):
    "Returns a list of smaller chunks of text"
    chars = len(text)
    blocks = chars / chop_size
    if chars % chop_size:
        blocks += 1
    return [text[i*chop_size:min(chars,(i+1)*chop_size)]
            for i in xrange(0, blocks)]

def chop_up ( text , chop_size = 5 ) : chars = len ( text ) blocks = chars / chop_size if chars % chop_size : blocks += 1 return [ text [ i * chop_size : min ( chars , ( i + 1 ) * chop_size ) ] for i in xrange ( 0 , blocks ) ]


Convenience method that works with all 2 . x versions of Python to determine whether or not something is listlike .

def isList(l):
    """Convenience method that works with all 2.x versions of Python
    to determine whether or not something is listlike."""
    return hasattr(l, '__iter__') \
           or (type(l) in (types.ListType, types.TupleType))

def isList ( l ) : return hasattr ( l , '__iter__' ) or ( type ( l ) in ( types . ListType , types . TupleType ) )


Convenience method that works with all 2 . x versions of Python to determine whether or not something is stringlike .

def isString(s):
    """Convenience method that works with all 2.x versions of Python
    to determine whether or not something is stringlike."""
    try:
        return isinstance(s, unicode) or isinstance(s, basestring)
    except NameError:
        return isinstance(s, str)

def isString ( s ) : try : return isinstance ( s , unicode ) or isinstance ( s , basestring ) except NameError : return isinstance ( s , str )


Turns a list of maps lists or scalars into a single map . Used to build the SELF_CLOSING_TAGS NESTABLE_TAGS and NESTING_RESET_TAGS maps out of lists and partial maps .

def buildTagMap(default, *args):
    """Turns a list of maps, lists, or scalars into a single map.
    Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and
    NESTING_RESET_TAGS maps out of lists and partial maps."""
    built = {}
    for portion in args:
        if hasattr(portion, 'items'):
            #It's a map. Merge it.
            for k,v in portion.items():
                built[k] = v
        elif isList(portion):
            #It's a list. Map each item to the default.
            for k in portion:
                built[k] = default
        else:
            #It's a scalar. Map it to the default.
            built[portion] = default
    return built


Sets up the initial relations between this element and other elements .

def setup(self, parent=None, previous=None):
        """Sets up the initial relations between this element and
        other elements."""
        self.parent = parent
        self.previous = previous
        self.next = None
        self.previousSibling = None
        self.nextSibling = None
        if self.parent and self.parent.contents:
            self.previousSibling = self.parent.contents[-1]
            self.previousSibling.nextSibling = self

def setup ( self , parent = None , previous = None ) : self . parent = parent self . previous = previous self . next = None self . previousSibling = None self . nextSibling = None if self . parent and self . parent . contents : self . previousSibling = self . parent . contents [ - 1 ] self . previousSibling . nextSibling = self


Destructively rips this element out of the tree .

def extract(self):
        """Destructively rips this element out of the tree."""
        if self.parent:
            try:
                self.parent.contents.remove(self)
            except ValueError:
                pass

        #Find the two elements that would be next to each other if
        #this element (and any children) hadn't been parsed. Connect
        #the two.
        lastChild = self._lastRecursiveChild()
        nextElement = lastChild.next

        if self.previous:
            self.previous.next = nextElement
        if nextElement:
            nextElement.previous = self.previous
        self.previous = None
        lastChild.next = None

        self.parent = None
        if self.previousSibling:
            self.previousSibling.nextSibling = self.nextSibling
        if self.nextSibling:
            self.nextSibling.previousSibling = self.previousSibling
        self.previousSibling = self.nextSibling = None
        return self


Finds the last element beneath this object to be parsed .

def _lastRecursiveChild(self):
        "Finds the last element beneath this object to be parsed."
        lastChild = self
        while hasattr(lastChild, 'contents') and lastChild.contents:
            lastChild = lastChild.contents[-1]
        return lastChild

def _lastRecursiveChild ( self ) : lastChild = self while hasattr ( lastChild , 'contents' ) and lastChild . contents : lastChild = lastChild . contents [ - 1 ] return lastChild


Appends the given tag to the contents of this tag .

def append(self, tag):
        """Appends the given tag to the contents of this tag."""
        self.insert(len(self.contents), tag)

def append ( self , tag ) : self . insert ( len ( self . contents ) , tag )


Returns the first item that matches the given criteria and appears after this Tag in the document .

def findNext(self, name=None, attrs={}, text=None, **kwargs):
        """Returns the first item that matches the given criteria and
        appears after this Tag in the document."""
        return self._findOne(self.findAllNext, name, attrs, text, **kwargs)

def findNext ( self , name = None , attrs = { } , text = None , * * kwargs ) : return self . _findOne ( self . findAllNext , name , attrs , text , * * kwargs )


Returns all items that match the given criteria and appear after this Tag in the document .

def findAllNext(self, name=None, attrs={}, text=None, limit=None,
                    **kwargs):
        """Returns all items that match the given criteria and appear
        after this Tag in the document."""
        return self._findAll(name, attrs, text, limit, self.nextGenerator,
                             **kwargs)

def findAllNext ( self , name = None , attrs = { } , text = None , limit = None , * * kwargs ) : return self . _findAll ( name , attrs , text , limit , self . nextGenerator , * * kwargs )


Returns the closest sibling to this Tag that matches the given criteria and appears after this Tag in the document .

def findNextSibling(self, name=None, attrs={}, text=None, **kwargs):
        """Returns the closest sibling to this Tag that matches the
        given criteria and appears after this Tag in the document."""
        return self._findOne(self.findNextSiblings, name, attrs, text,
                             **kwargs)

def findNextSibling ( self , name = None , attrs = { } , text = None , * * kwargs ) : return self . _findOne ( self . findNextSiblings , name , attrs , text , * * kwargs )


Returns the siblings of this Tag that match the given criteria and appear after this Tag in the document .

def findNextSiblings(self, name=None, attrs={}, text=None, limit=None,
                         **kwargs):
        """Returns the siblings of this Tag that match the given
        criteria and appear after this Tag in the document."""
        return self._findAll(name, attrs, text, limit,
                             self.nextSiblingGenerator, **kwargs)

def findNextSiblings ( self , name = None , attrs = { } , text = None , limit = None , * * kwargs ) : return self . _findAll ( name , attrs , text , limit , self . nextSiblingGenerator , * * kwargs )


Returns the first item that matches the given criteria and appears before this Tag in the document .

def findPrevious(self, name=None, attrs={}, text=None, **kwargs):
        """Returns the first item that matches the given criteria and
        appears before this Tag in the document."""
        return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs)

def findPrevious ( self , name = None , attrs = { } , text = None , * * kwargs ) : return self . _findOne ( self . findAllPrevious , name , attrs , text , * * kwargs )


Returns all items that match the given criteria and appear before this Tag in the document .

def findAllPrevious(self, name=None, attrs={}, text=None, limit=None,
                        **kwargs):
        """Returns all items that match the given criteria and appear
        before this Tag in the document."""
        return self._findAll(name, attrs, text, limit, self.previousGenerator,
                           **kwargs)

def findAllPrevious ( self , name = None , attrs = { } , text = None , limit = None , * * kwargs ) : return self . _findAll ( name , attrs , text , limit , self . previousGenerator , * * kwargs )


Returns the closest sibling to this Tag that matches the given criteria and appears before this Tag in the document .

def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs):
        """Returns the closest sibling to this Tag that matches the
        given criteria and appears before this Tag in the document."""
        return self._findOne(self.findPreviousSiblings, name, attrs, text,
                             **kwargs)

def findPreviousSibling ( self , name = None , attrs = { } , text = None , * * kwargs ) : return self . _findOne ( self . findPreviousSiblings , name , attrs , text , * * kwargs )


Returns the siblings of this Tag that match the given criteria and appear before this Tag in the document .

def findPreviousSiblings(self, name=None, attrs={}, text=None,
                             limit=None, **kwargs):
        """Returns the siblings of this Tag that match the given
        criteria and appear before this Tag in the document."""
        return self._findAll(name, attrs, text, limit,
                             self.previousSiblingGenerator, **kwargs)

def findPreviousSiblings ( self , name = None , attrs = { } , text = None , limit = None , * * kwargs ) : return self . _findAll ( name , attrs , text , limit , self . previousSiblingGenerator , * * kwargs )


Returns the closest parent of this Tag that matches the given criteria .

def findParent(self, name=None, attrs={}, **kwargs):
        """Returns the closest parent of this Tag that matches the given
        criteria."""
        # NOTE: We can't use _findOne because findParents takes a different
        # set of arguments.
        r = None
        l = self.findParents(name, attrs, 1)
        if l:
            r = l[0]
        return r


Returns the parents of this Tag that match the given criteria .

def findParents(self, name=None, attrs={}, limit=None, **kwargs):
        """Returns the parents of this Tag that match the given
        criteria."""

        return self._findAll(name, attrs, None, limit, self.parentGenerator,
                             **kwargs)

def findParents ( self , name = None , attrs = { } , limit = None , * * kwargs ) : return self . _findAll ( name , attrs , None , limit , self . parentGenerator , * * kwargs )


Iterates over a generator looking for things that match .

def _findAll(self, name, attrs, text, limit, generator, **kwargs):
        "Iterates over a generator looking for things that match."

        if isinstance(name, SoupStrainer):
            strainer = name
        else:
            # Build a SoupStrainer
            strainer = SoupStrainer(name, attrs, text, **kwargs)
        results = ResultSet(strainer)
        g = generator()
        while True:
            try:
                i = g.next()
            except StopIteration:
                break
            if i:
                found = strainer.search(i)
                if found:
                    results.append(found)
                    if limit and len(results) >= limit:
                        break
        return results

def
_findAll
(
self
,
name
,
attrs
,
text
,
limit
,
generator
,
*
*
kwargs
)
:
if
isinstance
(
name
,
SoupStrainer
)
:
strainer
=
name
else
:
# Build a SoupStrainer
strainer
=
SoupStrainer
(
name
,
attrs
,
text
,
*
*
kwargs
)
results
=
ResultSet
(
strainer
)
g
=
generator
(
)
while
True
:
try
:
i
=
g
.
next
(
)
except
StopIteration
:
break
if
i
:
found
=
strainer
.
search
(
i
)
if
found
:
results
.
append
(
found
)
if
limit
and
len
(
results
)
>=
limit
:
break
return
results


Encodes an object to a string in some encoding or to Unicode . .

def toEncoding(self, s, encoding=None):
        """Encodes an object to a string in some encoding, or to Unicode.
        ."""
        if isinstance(s, unicode):
            if encoding:
                s = s.encode(encoding)
        elif isinstance(s, str):
            if encoding:
                s = s.encode(encoding)
            else:
                s = unicode(s)
        else:
            if encoding:
                s  = self.toEncoding(str(s), encoding)
            else:
                s = unicode(s)
        return s

def toEncoding ( self , s , encoding = None ) : if isinstance ( s , unicode ) : if encoding : s = s . encode ( encoding ) elif isinstance ( s , str ) : if encoding : s = s . encode ( encoding ) else : s = unicode ( s ) else : if encoding : s = self . toEncoding ( str ( s ) , encoding ) else : s = unicode ( s ) return s


def __new__(cls, value):
        """Create a new NavigableString.

        When unpickling a NavigableString, this method is called with
        the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be
        passed in to the superclass's __new__ or the superclass won't know
        how to handle non-ASCII characters.
        """
        if isinstance(value, unicode):
            return unicode.__new__(cls, value)
        return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)

def __new__ ( cls , value ) : if isinstance ( value , unicode ) : return unicode . __new__ ( cls , value ) return unicode . __new__ ( cls , value , DEFAULT_OUTPUT_ENCODING )


text . string gives you text . This is for backwards compatibility for Navigable * String but for CData * it lets you get the string without the CData wrapper .

def __getattr__(self, attr):
        """text.string gives you text. This is for backwards
        compatibility for Navigable*String, but for CData* it lets you
        get the string without the CData wrapper."""
        if attr == 'string':
            return self
        else:
            raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)

def
__getattr__
(
self
,
attr
)
:
if
attr
==
'string'
:
return
self
else
:
raise
AttributeError
,
"'%s' object has no attribute '%s'"
%
(
self
.
__class__
.
__name__
,
attr
)


def _invert(h):
        "Cheap function to invert a hash."
        i = {}
        for k,v in h.items():
            i[v] = k
        return i

def _invert ( h ) : i = { } for k , v in h . items ( ) : i [ v ] = k return i


Used in a call to re . sub to replace HTML XML and numeric entities with the appropriate Unicode characters . If HTML entities are being converted any unrecognized entities are escaped .

def _convertEntities(self, match):
        """Used in a call to re.sub to replace HTML, XML, and numeric
        entities with the appropriate Unicode characters. If HTML
        entities are being converted, any unrecognized entities are
        escaped."""
        x = match.group(1)
        if self.convertHTMLEntities and x in name2codepoint:
            return unichr(name2codepoint[x])
        elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS:
            if self.convertXMLEntities:
                return self.XML_ENTITIES_TO_SPECIAL_CHARS[x]
            else:
                return u'&%s;' % x
        elif len(x) > 0 and x[0] == '#':
            # Handle numeric entities
            if len(x) > 1 and x[1] == 'x':
                return unichr(int(x[2:], 16))
            else:
                return unichr(int(x[1:]))

        elif self.escapeUnrecognizedEntities:
            return u'&%s;' % x
        else:
            return u'&%s;' % x

def
_convertEntities
(
self
,
match
)
:
x
=
match
.
group
(
1
)
if
self
.
convertHTMLEntities
and
x
in
name2codepoint
:
return
unichr
(
name2codepoint
[
x
]
)
elif
x
in
self
.
XML_ENTITIES_TO_SPECIAL_CHARS
:
if
self
.
convertXMLEntities
:
return
self
.
XML_ENTITIES_TO_SPECIAL_CHARS
[
x
]
else
:
return
u'&%s;'
%
x
elif
len
(
x
)
>
0
and
x
[
0
]
==
'#'
:
# Handle numeric entities
if
len
(
x
)
>
1
and
x
[
1
]
==
'x'
:
return
unichr
(
int
(
x
[
2
:
]
,
16
)
)
else
:
return
unichr
(
int
(
x
[
1
:
]
)
)
elif
self
.
escapeUnrecognizedEntities
:
return
u'&%s;'
%
x
else
:
return
u'&%s;'
%
x


def __init__(self, parser, name, attrs=None, parent=None,
                 previous=None):
        "Basic constructor."

        # We don't actually store the parser object: that lets extracted
        # chunks be garbage-collected
        self.parserClass = parser.__class__
        self.isSelfClosing = parser.isSelfClosingTag(name)
        self.name = name
        if attrs == None:
            attrs = []
        self.attrs = attrs
        self.contents = []
        self.setup(parent, previous)
        self.hidden = False
        self.containsSubstitutions = False
        self.convertHTMLEntities = parser.convertHTMLEntities
        self.convertXMLEntities = parser.convertXMLEntities
        self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities

        # Convert any HTML, XML, or numeric entities in the attribute values.
        convert = lambda(k, val): (k,
                                   re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);",
                                          self._convertEntities,
                                          val))
        self.attrs = map(convert, self.attrs)


Returns the value of the key attribute for the tag or the value given for default if it doesn t have that attribute .

def get(self, key, default=None):
        """Returns the value of the 'key' attribute for the tag, or
        the value given for 'default' if it doesn't have that
        attribute."""
        return self._getAttrMap().get(key, default)

def get ( self , key , default = None ) : return self . _getAttrMap ( ) . get ( key , default )


tag [ key ] returns the value of the key attribute for the tag and throws an exception if it s not there .

def __getitem__(self, key):
        """tag[key] returns the value of the 'key' attribute for the tag,
        and throws an exception if it's not there."""
        return self._getAttrMap()[key]

def __getitem__ ( self , key ) : return self . _getAttrMap ( ) [ key ]


Iterating over a tag iterates over its contents .

def __iter__(self):
        "Iterating over a tag iterates over its contents."
        return iter(self.contents)

def __iter__ ( self ) : return iter ( self . contents )


The length of a tag is the length of its list of contents .

def __len__(self):
        "The length of a tag is the length of its list of contents."
        return len(self.contents)

def __len__ ( self ) : return len ( self . contents )


A tag is non - None even if it has no contents .

def __nonzero__(self):
        "A tag is non-None even if it has no contents."
        return True


Setting tag [ key ] sets the value of the key attribute for the tag .

def __setitem__(self, key, value):
        """Setting tag[key] sets the value of the 'key' attribute for the
        tag."""
        self._getAttrMap()
        self.attrMap[key] = value
        found = False
        for i in range(0, len(self.attrs)):
            if self.attrs[i][0] == key:
                self.attrs[i] = (key, value)
                found = True
        if not found:
            self.attrs.append((key, value))
        self._getAttrMap()[key] = value

def __setitem__ ( self , key , value ) : self . _getAttrMap ( ) self . attrMap [ key ] = value found = False for i in range ( 0 , len ( self . attrs ) ) : if self . attrs [ i ] [ 0 ] == key : self . attrs [ i ] = ( key , value ) found = True if not found : self . attrs . append ( ( key , value ) ) self . _getAttrMap ( ) [ key ] = value


Deleting tag [ key ] deletes all key attributes for the tag .

def __delitem__(self, key):
        "Deleting tag[key] deletes all 'key' attributes for the tag."
        for item in self.attrs:
            if item[0] == key:
                self.attrs.remove(item)
                #We don't break because bad HTML can define the same
                #attribute multiple times.
            self._getAttrMap()
            if self.attrMap.has_key(key):
                del self.attrMap[key]


Calling a tag like a function is the same as calling its findAll () method . Eg . tag ( a ) returns a list of all the A tags found within this tag .

def __call__(self, *args, **kwargs):
        """Calling a tag like a function is the same as calling its
        findAll() method. Eg. tag('a') returns a list of all the A tags
        found within this tag."""
        return apply(self.findAll, args, kwargs)

def __call__ ( self , * args , * * kwargs ) : return apply ( self . findAll , args , kwargs )


Returns true iff this tag has the same name the same attributes and the same contents ( recursively ) as the given tag .

def __eq__(self, other):
        """Returns true iff this tag has the same name, the same attributes,
        and the same contents (recursively) as the given tag.

        NOTE: right now this will return false if two tags have the
        same attributes in a different order. Should this be fixed?"""
        if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
            return False
        for i in range(0, len(self.contents)):
            if self.contents[i] != other.contents[i]:
                return False
        return True

def __eq__ ( self , other ) : if not hasattr ( other , 'name' ) or not hasattr ( other , 'attrs' ) or not hasattr ( other , 'contents' ) or self . name != other . name or self . attrs != other . attrs or len ( self ) != len ( other ) : return False for i in range ( 0 , len ( self . contents ) ) : if self . contents [ i ] != other . contents [ i ] : return False return True


Returns true iff this tag is not identical to the other tag as defined in __eq__ .

def __ne__(self, other):
        """Returns true iff this tag is not identical to the other tag,
        as defined in __eq__."""
        return not self == other

def __ne__ ( self , other ) : return not self == other


def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
        """Renders this tag as a string."""
        return self.__str__(encoding)

def __repr__ ( self , encoding = DEFAULT_OUTPUT_ENCODING ) : return self . __str__ ( encoding )


Used with a regular expression to substitute the appropriate XML entity for an XML special character .

def _sub_entity(self, x):
        """Used with a regular expression to substitute the
        appropriate XML entity for an XML special character."""
        return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";"

def _sub_entity ( self , x ) : return "&" + self . XML_SPECIAL_CHARS_TO_ENTITIES [ x . group ( 0 ) [ 0 ] ] + ";"


Returns a string or Unicode representation of this tag and its contents . To get Unicode pass None for encoding .

def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING,
                prettyPrint=False, indentLevel=0):
        """Returns a string or Unicode representation of this tag and
        its contents. To get Unicode, pass None for encoding.

        NOTE: since Python's HTML parser consumes whitespace, this
        method is not certain to reproduce the whitespace present in
        the original string."""

        encodedName = self.toEncoding(self.name, encoding)

        attrs = []
        if self.attrs:
            for key, val in self.attrs:
                fmt = '%s="%s"'
                if isString(val):
                    if self.containsSubstitutions and '%SOUP-ENCODING%' in val:
                        val = self.substituteEncoding(val, encoding)

                    # The attribute value either:
                    #
                    # * Contains no embedded double quotes or single quotes.
                    #   No problem: we enclose it in double quotes.
                    # * Contains embedded single quotes. No problem:
                    #   double quotes work here too.
                    # * Contains embedded double quotes. No problem:
                    #   we enclose it in single quotes.
                    # * Embeds both single _and_ double quotes. This
                    #   can't happen naturally, but it can happen if
                    #   you modify an attribute value after parsing
                    #   the document. Now we have a bit of a
                    #   problem. We solve it by enclosing the
                    #   attribute in single quotes, and escaping any
                    #   embedded single quotes to XML entities.
                    if '"' in val:
                        fmt = "%s='%s'"
                        if "'" in val:
                            # TODO: replace with apos when
                            # appropriate.
                            val = val.replace("'", "&squot;")

                    # Now we're okay w/r/t quotes. But the attribute
                    # value might also contain angle brackets, or
                    # ampersands that aren't part of entities. We need
                    # to escape those to XML entities too.
                    val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val)

                attrs.append(fmt % (self.toEncoding(key, encoding),
                                    self.toEncoding(val, encoding)))
        close = ''
        closeTag = ''
        if self.isSelfClosing:
            close = ' /'
        else:
            closeTag = '</%s>' % encodedName

        indentTag, indentContents = 0, 0
        if prettyPrint:
            indentTag = indentLevel
            space = (' ' * (indentTag-1))
            indentContents = indentTag + 1
        contents = self.renderContents(encoding, prettyPrint, indentContents)
        if self.hidden:
            s = contents
        else:
            s = []
            attributeString = ''
            if attrs:
                attributeString = ' ' + ' '.join(attrs)
            if prettyPrint:
                s.append(space)
            s.append('<%s%s%s>' % (encodedName, attributeString, close))
            if prettyPrint:
                s.append("\n")
            s.append(contents)
            if prettyPrint and contents and contents[-1] != "\n":
                s.append("\n")
            if prettyPrint and closeTag:
                s.append(space)
            s.append(closeTag)
            if prettyPrint and closeTag and self.nextSibling:
                s.append("\n")
            s = ''.join(s)
        return s

#   the document. Now we have a bit of a

#   problem. We solve it by enclosing the


Recursively destroys the contents of this tree .

def decompose(self):
        """Recursively destroys the contents of this tree."""
        contents = [i for i in self.contents]
        for i in contents:
            if isinstance(i, Tag):
                i.decompose()
            else:
                i.extract()
        self.extract()

def decompose ( self ) : contents = [ i for i in self . contents ] for i in contents : if isinstance ( i , Tag ) : i . decompose ( ) else : i . extract ( ) self . extract ( )


Renders the contents of this tag as a string in the given encoding . If encoding is None returns a Unicode string ..

def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
                       prettyPrint=False, indentLevel=0):
        """Renders the contents of this tag as a string in the given
        encoding. If encoding is None, returns a Unicode string.."""
        s=[]
        for c in self:
            text = None
            if isinstance(c, NavigableString):
                text = c.__str__(encoding)
            elif isinstance(c, Tag):
                s.append(c.__str__(encoding, prettyPrint, indentLevel))
            if text and prettyPrint:
                text = text.strip()
            if text:
                if prettyPrint:
                    s.append(" " * (indentLevel-1))
                s.append(text)
                if prettyPrint:
                    s.append("\n")
        return ''.join(s)

def
renderContents
(
self
,
encoding
=
DEFAULT_OUTPUT_ENCODING
,
prettyPrint
=
False
,
indentLevel
=
0
)
:
s
=
[
]
for
c
in
self
:
text
=
if
isinstance
(
c
,
NavigableString
)
:
text
=
c
.
__str__
(
encoding
)
elif
isinstance
(
c
,
Tag
)
:
s
.
append
(
c
.
__str__
(
encoding
,
prettyPrint
,
indentLevel
)
)
if
text
and
prettyPrint
:
text
=
text
.
strip
(
)
if
text
:
if
prettyPrint
:
s
.
append
(
" "
*
(
indentLevel
1
)
)
s
.
append
(
text
)
if
prettyPrint
:
s
.
append
(
"\n"
)
return
''
.
join
(
s
)


Return only the first child of this Tag matching the given criteria .

def find(self, name=None, attrs={}, recursive=True, text=None,
             **kwargs):
        """Return only the first child of this Tag matching the given
        criteria."""
        r = None
        l = self.findAll(name, attrs, recursive, text, 1, **kwargs)
        if l:
            r = l[0]
        return r

def find ( self , name = None , attrs = { } , recursive = True , text = None , * * kwargs ) : r = None l = self . findAll ( name , attrs , recursive , text , 1 , * * kwargs ) if l : r = l [ 0 ] return r


Extracts a list of Tag objects that match the given criteria . You can specify the name of the Tag and any attributes you want the Tag to have .

def findAll(self, name=None, attrs={}, recursive=True, text=None,
                limit=None, **kwargs):
        """Extracts a list of Tag objects that match the given
        criteria.  You can specify the name of the Tag and any
        attributes you want the Tag to have.

        The value of a key-value pair in the 'attrs' map can be a
        string, a list of strings, a regular expression object, or a
        callable that takes a string and returns whether or not the
        string matches for some custom definition of 'matches'. The
        same is true of the tag name."""
        generator = self.recursiveChildGenerator
        if not recursive:
            generator = self.childGenerator
        return self._findAll(name, attrs, text, limit, generator, **kwargs)

def findAll ( self , name = None , attrs = { } , recursive = True , text = None , limit = None , * * kwargs ) : generator = self . recursiveChildGenerator if not recursive : generator = self . childGenerator return self . _findAll ( name , attrs , text , limit , generator , * * kwargs )


Initializes a map representation of this tag s attributes if not already initialized .

def _getAttrMap(self):
        """Initializes a map representation of this tag's attributes,
        if not already initialized."""
        if not getattr(self, 'attrMap'):
            self.attrMap = {}
            for (key, value) in self.attrs:
                self.attrMap[key] = value
        return self.attrMap

def _getAttrMap ( self ) : if not getattr ( self , 'attrMap' ) : self . attrMap = { } for ( key , value ) in self . attrs : self . attrMap [ key ] = value return self . attrMap


The Soup object is initialized as the root tag and the provided markup ( which can be a string or a file - like object ) is fed into the underlying parser .

def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None,
                 markupMassage=True, smartQuotesTo=XML_ENTITIES,
                 convertEntities=None, selfClosingTags=None, isHTML=False):
        """The Soup object is initialized as the 'root tag', and the
        provided markup (which can be a string or a file-like object)
        is fed into the underlying parser.

        sgmllib will process most bad HTML, and the BeautifulSoup
        class has some tricks for dealing with some HTML that kills
        sgmllib, but Beautiful Soup can nonetheless choke or lose data
        if your data uses self-closing tags or declarations
        incorrectly.

        By default, Beautiful Soup uses regexes to sanitize input,
        avoiding the vast majority of these problems. If the problems
        don't apply to you, pass in False for markupMassage, and
        you'll get better performance.

        The default parser massage techniques fix the two most common
        instances of invalid HTML that choke sgmllib:

         <br/> (No space between name of closing tag and tag close)
         <! --Comment--> (Extraneous whitespace in declaration)

        You can pass in a custom list of (RE object, replace method)
        tuples to get Beautiful Soup to scrub your input the way you
        want."""

        self.parseOnlyThese = parseOnlyThese
        self.fromEncoding = fromEncoding
        self.smartQuotesTo = smartQuotesTo
        self.convertEntities = convertEntities
        # Set the rules for how we'll deal with the entities we
        # encounter
        if self.convertEntities:
            # It doesn't make sense to convert encoded characters to
            # entities even while you're converting entities to Unicode.
            # Just convert it all to Unicode.
            self.smartQuotesTo = None
            if convertEntities == self.HTML_ENTITIES:
                self.convertXMLEntities = False
                self.convertHTMLEntities = True
                self.escapeUnrecognizedEntities = True
            elif convertEntities == self.XHTML_ENTITIES:
                self.convertXMLEntities = True
                self.convertHTMLEntities = True
                self.escapeUnrecognizedEntities = False
            elif convertEntities == self.XML_ENTITIES:
                self.convertXMLEntities = True
                self.convertHTMLEntities = False
                self.escapeUnrecognizedEntities = False
        else:
            self.convertXMLEntities = False
            self.convertHTMLEntities = False
            self.escapeUnrecognizedEntities = False

        self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags)
        SGMLParser.__init__(self)

        if hasattr(markup, 'read'):        # It's a file-type object.
            markup = markup.read()
        self.markup = markup
        self.markupMassage = markupMassage
        try:
            self._feed(isHTML=isHTML)
        except StopParsing:
            pass
        self.markup = None


This method fixes a bug in Python s SGMLParser .

def convert_charref(self, name):
        """This method fixes a bug in Python's SGMLParser."""
        try:
            n = int(name)
        except ValueError:
            return
        if not 0 <= n <= 127 : # ASCII ends at 127, not 255
            return
        return self.convert_codepoint(n)

def
convert_charref
(
self
,
name
)
:
try
:
n
=
int
(
name
)
except
ValueError
:
return
if
not
0
<=
n
<=
127
:
# ASCII ends at 127, not 255
return
return
self
.
convert_codepoint
(
n
)


This method routes method call requests to either the SGMLParser superclass or the Tag superclass depending on the method name .

def __getattr__(self, methodName):
        """This method routes method call requests to either the SGMLParser
        superclass or the Tag superclass, depending on the method name."""
        #print "__getattr__ called on %s.%s" % (self.__class__, methodName)

        if methodName.find('start_') == 0 or methodName.find('end_') == 0 \
               or methodName.find('do_') == 0:
            return SGMLParser.__getattr__(self, methodName)
        elif methodName.find('__') != 0:
            return Tag.__getattr__(self, methodName)
        else:
            raise AttributeError


Returns true iff the given string is the name of a self - closing tag according to this parser .

def isSelfClosingTag(self, name):
        """Returns true iff the given string is the name of a
        self-closing tag according to this parser."""
        return self.SELF_CLOSING_TAGS.has_key(name) \
               or self.instanceSelfClosingTags.has_key(name)

def isSelfClosingTag ( self , name ) : return self . SELF_CLOSING_TAGS . has_key ( name ) or self . instanceSelfClosingTags . has_key ( name )


Pops the tag stack up to and including the most recent instance of the given tag . If inclusivePop is false pops the tag stack up to but * not * including the most recent instqance of the given tag .

def _popToTag(self, name, inclusivePop=True):
        """Pops the tag stack up to and including the most recent
        instance of the given tag. If inclusivePop is false, pops the tag
        stack up to but *not* including the most recent instqance of
        the given tag."""
        #print "Popping to %s" % name
        if name == self.ROOT_TAG_NAME:
            return

        numPops = 0
        mostRecentTag = None
        for i in range(len(self.tagStack)-1, 0, -1):
            if name == self.tagStack[i].name:
                numPops = len(self.tagStack)-i
                break
        if not inclusivePop:
            numPops = numPops - 1

        for i in range(0, numPops):
            mostRecentTag = self.popTag()
        return mostRecentTag

def
_popToTag
(
self
,
name
,
inclusivePop
=
True
)
:
#print "Popping to %s" % name
if
name
==
self
.
ROOT_TAG_NAME
:
return
numPops
=
0
mostRecentTag
=
for
i
in
range
(
len
(
self
.
tagStack
)
1
,
0
,
1
)
:
if
name
==
self
.
tagStack
[
i
]
.
name
:
numPops
=
len
(
self
.
tagStack
)
i
break
if
not
inclusivePop
:
numPops
=
numPops
1
for
i
in
range
(
0
,
numPops
)
:
mostRecentTag
=
self
.
popTag
(
)
return
mostRecentTag


We need to pop up to the previous tag of this type unless one of this tag s nesting reset triggers comes between this tag and the previous tag of this type OR unless this tag is a generic nesting trigger and another generic nesting trigger comes between this tag and the previous tag of this type .

def _smartPop(self, name):

        """We need to pop up to the previous tag of this type, unless
        one of this tag's nesting reset triggers comes between this
        tag and the previous tag of this type, OR unless this tag is a
        generic nesting trigger and another generic nesting trigger
        comes between this tag and the previous tag of this type.

        Examples:

FooBar *
* should pop to 'p', not 'b'.

Foo
Bar *
* should pop to 'table', not 'p'.

Foo

Bar *
* should pop to 'tr', not 'p'.

 *
* should pop to 'ul', not the first 'li'.

 *
* should pop to 'table', not the first 'tr'

 ** should pop to 'tr', not the first 'td'
        """

        nestingResetTriggers = self.NESTABLE_TAGS.get(name)
        isNestable = nestingResetTriggers != None
        isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
        popTo = None
        inclusive = True
        for i in range(len(self.tagStack)-1, 0, -1):
            p = self.tagStack[i]
            if (not p or p.name == name) and not isNestable:
                #Non-nestable tags get popped to the top or to their
                #last occurance.
                popTo = name
                break
            if (nestingResetTriggers != None
                and p.name in nestingResetTriggers) \
                or (nestingResetTriggers == None and isResetNesting
                    and self.RESET_NESTING_TAGS.has_key(p.name)):

                #If we encounter one of the nesting reset triggers
                #peculiar to this tag, or we encounter another tag
                #that causes nesting to reset, pop up to but not
                #including that tag.
                popTo = p.name
                inclusive = False
                break
            p = p.parent
        if popTo:
            self._popToTag(popTo, inclusive)


Adds a certain piece of text to the tree as a NavigableString subclass .

def _toStringSubclass(self, text, subclass):
        """Adds a certain piece of text to the tree as a NavigableString
        subclass."""
        self.endData()
        self.handle_data(text)
        self.endData(subclass)

def _toStringSubclass ( self , text , subclass ) : self . endData ( ) self . handle_data ( text ) self . endData ( subclass )


Handle a processing instruction as a ProcessingInstruction object possibly one with a %SOUP - ENCODING% slot into which an encoding will be plugged later .

def handle_pi(self, text):
        """Handle a processing instruction as a ProcessingInstruction
        object, possibly one with a %SOUP-ENCODING% slot into which an
        encoding will be plugged later."""
        if text[:3] == "xml":
            text = u"xml version='1.0' encoding='%SOUP-ENCODING%'"
        self._toStringSubclass(text, ProcessingInstruction)


def handle_comment(self, text):
        "Handle comments as Comment objects."
        self._toStringSubclass(text, Comment)

def handle_comment ( self , text ) : self . _toStringSubclass ( text , Comment )


def handle_charref(self, ref):
        "Handle character references as data."
        if self.convertEntities:
            data = unichr(int(ref))
        else:
            data = '&#%s;' % ref
        self.handle_data(data)

def handle_charref ( self , ref ) : if self . convertEntities : data = unichr ( int ( ref ) ) else : data = '&#%s;' % ref self . handle_data ( data )


Handle entity references as data possibly converting known HTML and / or XML entity references to the corresponding Unicode characters .

def handle_entityref(self, ref):
        """Handle entity references as data, possibly converting known
        HTML and/or XML entity references to the corresponding Unicode
        characters."""
        data = None
        if self.convertHTMLEntities:
            try:
                data = unichr(name2codepoint[ref])
            except KeyError:
                pass

        if not data and self.convertXMLEntities:
                data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref)

        if not data and self.convertHTMLEntities and \
            not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref):
                # TODO: We've got a problem here. We're told this is
                # an entity reference, but it's not an XML entity
                # reference or an HTML entity reference. Nonetheless,
                # the logical thing to do is to pass it through as an
                # unrecognized entity reference.
                #
                # Except: when the input is "&carol;" this function
                # will be called with input "carol". When the input is
                # "AT&T", this function will be called with input
                # "T". We have no way of knowing whether a semicolon
                # was present originally, so we don't know whether
                # this is an unknown entity or just a misplaced
                # ampersand.
                #
                # The more common case is a misplaced ampersand, so I
                # escape the ampersand and omit the trailing semicolon.
                data = "&%s" % ref
        if not data:
            # This case is different from the one above, because we
            # haven't already gone through a supposedly comprehensive
            # mapping of entities to Unicode characters. We might not
            # have gone through any mapping at all. So the chances are
            # very high that this is a real entity, and not a
            # misplaced ampersand.
            data = "&%s;" % ref
        self.handle_data(data)


Handle DOCTYPEs and the like as Declaration objects .

def handle_decl(self, data):
        "Handle DOCTYPEs and the like as Declaration objects."
        self._toStringSubclass(data, Declaration)

def handle_decl ( self , data ) : self . _toStringSubclass ( data , Declaration )


Treat a bogus SGML declaration as raw data . Treat a CDATA declaration as a CData object .

def parse_declaration(self, i):
        """Treat a bogus SGML declaration as raw data. Treat a CDATA
        declaration as a CData object."""
        j = None
        if self.rawdata[i:i+9] == '<![CDATA[':
             k = self.rawdata.find(']]>', i)
             if k == -1:
                 k = len(self.rawdata)
             data = self.rawdata[i+9:k]
             j = k+3
             self._toStringSubclass(data, CData)
        else:
            try:
                j = SGMLParser.parse_declaration(self, i)
            except SGMLParseError:
                toHandle = self.rawdata[i:]
                self.handle_data(toHandle)
                j = i + len(toHandle)
        return j

def parse_declaration ( self , i ) : j = None if self . rawdata [ i : i + 9 ] == '<![CDATA[' : k = self . rawdata . find ( ']]>' , i ) if k == - 1 : k = len ( self . rawdata ) data = self . rawdata [ i + 9 : k ] j = k + 3 self . _toStringSubclass ( data , CData ) else : try : j = SGMLParser . parse_declaration ( self , i ) except SGMLParseError : toHandle = self . rawdata [ i : ] self . handle_data ( toHandle ) j = i + len ( toHandle ) return j


Beautiful Soup can detect a charset included in a META tag try to convert the document to that charset and re - parse the document from the beginning .

def start_meta(self, attrs):
        """Beautiful Soup can detect a charset included in a META tag,
        try to convert the document to that charset, and re-parse the
        document from the beginning."""
        httpEquiv = None
        contentType = None
        contentTypeIndex = None
        tagNeedsEncodingSubstitution = False

        for i in range(0, len(attrs)):
            key, value = attrs[i]
            key = key.lower()
            if key == 'http-equiv':
                httpEquiv = value
            elif key == 'content':
                contentType = value
                contentTypeIndex = i

        if httpEquiv and contentType: # It's an interesting meta tag.
            match = self.CHARSET_RE.search(contentType)
            if match:
                if (self.declaredHTMLEncoding is not None or
                    self.originalEncoding == self.fromEncoding):
                    # An HTML encoding was sniffed while converting
                    # the document to Unicode, or an HTML encoding was
                    # sniffed during a previous pass through the
                    # document, or an encoding was specified
                    # explicitly and it worked. Rewrite the meta tag.
                    def rewrite(match):
                        return match.group(1) + "%SOUP-ENCODING%"
                    newAttr = self.CHARSET_RE.sub(rewrite, contentType)
                    attrs[contentTypeIndex] = (attrs[contentTypeIndex][0],
                                               newAttr)
                    tagNeedsEncodingSubstitution = True
                else:
                    # This is our first pass through the document.
                    # Go through it again with the encoding information.
                    newCharset = match.group(3)
                    if newCharset and newCharset != self.originalEncoding:
                        self.declaredHTMLEncoding = newCharset
                        self._feed(self.declaredHTMLEncoding)
                        raise StopParsing
                    pass
        tag = self.unknown_starttag("meta", attrs)
        if tag and tagNeedsEncodingSubstitution:
            tag.containsSubstitutions = True


Changes a MS smart quote character to an XML or HTML entity .

def _subMSChar(self, orig):
        """Changes a MS smart quote character to an XML or HTML
        entity."""
        sub = self.MS_CHARS.get(orig)
        if type(sub) == types.TupleType:
            if self.smartQuotesTo == 'xml':
                sub = '&#x%s;' % sub[1]
            else:
                sub = '&%s;' % sub[0]
        return sub

def _subMSChar ( self , orig ) : sub = self . MS_CHARS . get ( orig ) if type ( sub ) == types . TupleType : if self . smartQuotesTo == 'xml' : sub = '&#x%s;' % sub [ 1 ] else : sub = '&%s;' % sub [ 0 ] return sub


Given a string and its encoding decodes the string into Unicode . %encoding is a string recognized by encodings . aliases

def _toUnicode(self, data, encoding):
        '''Given a string and its encoding, decodes the string into Unicode.
        %encoding is a string recognized by encodings.aliases'''

        # strip Byte Order Mark (if present)
        if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
               and (data[2:4] != '\x00\x00'):
            encoding = 'utf-16be'
            data = data[2:]
        elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
                 and (data[2:4] != '\x00\x00'):
            encoding = 'utf-16le'
            data = data[2:]
        elif data[:3] == '\xef\xbb\xbf':
            encoding = 'utf-8'
            data = data[3:]
        elif data[:4] == '\x00\x00\xfe\xff':
            encoding = 'utf-32be'
            data = data[4:]
        elif data[:4] == '\xff\xfe\x00\x00':
            encoding = 'utf-32le'
            data = data[4:]
        newdata = unicode(data, encoding)
        return newdata

def
_toUnicode
(
self
,
data
,
encoding
)
:
# strip Byte Order Mark (if present)
if
(
len
(
data
)
>=
4
)
and
(
data
[
:
2
]
==
'\xfe\xff'
)
and
(
data
[
2
:
4
]
!=
'\x00\x00'
)
:
encoding
=
'utf-16be'
data
=
data
[
2
:
]
elif
(
len
(
data
)
>=
4
)
and
(
data
[
:
2
]
==
'\xff\xfe'
)
and
(
data
[
2
:
4
]
!=
'\x00\x00'
)
:
encoding
=
'utf-16le'
data
=
data
[
2
:
]
elif
data
[
:
3
]
==
'\xef\xbb\xbf'
:
encoding
=
'utf-8'
data
=
data
[
3
:
]
elif
data
[
:
4
]
==
'\x00\x00\xfe\xff'
:
encoding
=
'utf-32be'
data
=
data
[
4
:
]
elif
data
[
:
4
]
==
'\xff\xfe\x00\x00'
:
encoding
=
'utf-32le'
data
=
data
[
4
:
]
newdata
=
unicode
(
data
,
encoding
)
return
newdata


Given a document tries to detect its XML encoding .

def _detectEncoding(self, xml_data, isHTML=False):
        """Given a document, tries to detect its XML encoding."""
        xml_encoding = sniffed_xml_encoding = None
        try:
            if xml_data[:4] == '\x4c\x6f\xa7\x94':
                # EBCDIC
                xml_data = self._ebcdic_to_ascii(xml_data)
            elif xml_data[:4] == '\x00\x3c\x00\x3f':
                # UTF-16BE
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') \
                     and (xml_data[2:4] != '\x00\x00'):
                # UTF-16BE with BOM
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
            elif xml_data[:4] == '\x3c\x00\x3f\x00':
                # UTF-16LE
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and \
                     (xml_data[2:4] != '\x00\x00'):
                # UTF-16LE with BOM
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
            elif xml_data[:4] == '\x00\x00\x00\x3c':
                # UTF-32BE
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
            elif xml_data[:4] == '\x3c\x00\x00\x00':
                # UTF-32LE
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
            elif xml_data[:4] == '\x00\x00\xfe\xff':
                # UTF-32BE with BOM
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
            elif xml_data[:4] == '\xff\xfe\x00\x00':
                # UTF-32LE with BOM
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
            elif xml_data[:3] == '\xef\xbb\xbf':
                # UTF-8 with BOM
                sniffed_xml_encoding = 'utf-8'
                xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
            else:
                sniffed_xml_encoding = 'ascii'
                pass
        except:
            xml_encoding_match = None
        xml_encoding_match = re.compile(
            '^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data)
        if not xml_encoding_match and isHTML:
            regexp = re.compile('<\s*meta[^>]+charset=([^>]*?)[;\'">]', re.I)
            xml_encoding_match = regexp.search(xml_data)
        if xml_encoding_match is not None:
            xml_encoding = xml_encoding_match.groups()[0].lower()
            if isHTML:
                self.declaredHTMLEncoding = xml_encoding
            if sniffed_xml_encoding and \
               (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
                                 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
                                 'utf-16', 'utf-32', 'utf_16', 'utf_32',
                                 'utf16', 'u16')):
                xml_encoding = sniffed_xml_encoding
        return xml_data, xml_encoding, sniffed_xml_encoding

def
_detectEncoding
(
self
,
xml_data
,
isHTML
=
False
)
:
xml_encoding
=
sniffed_xml_encoding
=
try
:
if
xml_data
[
:
4
]
==
'\x4c\x6f\xa7\x94'
:
# EBCDIC
xml_data
=
self
.
_ebcdic_to_ascii
(
xml_data
)
elif
xml_data
[
:
4
]
==
'\x00\x3c\x00\x3f'
:
# UTF-16BE
sniffed_xml_encoding
=
'utf-16be'
xml_data
=
unicode
(
xml_data
,
'utf-16be'
)
.
encode
(
'utf-8'
)
elif
(
len
(
xml_data
)
>=
4
)
and
(
xml_data
[
:
2
]
==
'\xfe\xff'
)
and
(
xml_data
[
2
:
4
]
!=
'\x00\x00'
)
:
# UTF-16BE with BOM
sniffed_xml_encoding
=
'utf-16be'
xml_data
=
unicode
(
xml_data
[
2
:
]
,
'utf-16be'
)
.
encode
(
'utf-8'
)
elif
xml_data
[
:
4
]
==
'\x3c\x00\x3f\x00'
:
# UTF-16LE
sniffed_xml_encoding
=
'utf-16le'
xml_data
=
unicode
(
xml_data
,
'utf-16le'
)
.
encode
(
'utf-8'
)
elif
(
len
(
xml_data
)
>=
4
)
and
(
xml_data
[
:
2
]
==
'\xff\xfe'
)
and
(
xml_data
[
2
:
4
]
!=
'\x00\x00'
)
:
# UTF-16LE with BOM
sniffed_xml_encoding
=
'utf-16le'
xml_data
=
unicode
(
xml_data
[
2
:
]
,
'utf-16le'
)
.
encode
(
'utf-8'
)
elif
xml_data
[
:
4
]
==
'\x00\x00\x00\x3c'
:
# UTF-32BE
sniffed_xml_encoding
=
'utf-32be'
xml_data
=
unicode
(
xml_data
,
'utf-32be'
)
.
encode
(
'utf-8'
)
elif
xml_data
[
:
4
]
==
'\x3c\x00\x00\x00'
:
# UTF-32LE
sniffed_xml_encoding
=
'utf-32le'
xml_data
=
unicode
(
xml_data
,
'utf-32le'
)
.
encode
(
'utf-8'
)
elif
xml_data
[
:
4
]
==
'\x00\x00\xfe\xff'
:
# UTF-32BE with BOM
sniffed_xml_encoding
=
'utf-32be'
xml_data
=
unicode
(
xml_data
[
4
:
]
,
'utf-32be'
)
.
encode
(
'utf-8'
)
elif
xml_data
[
:
4
]
==
'\xff\xfe\x00\x00'
:
# UTF-32LE with BOM
sniffed_xml_encoding
=
'utf-32le'
xml_data
=
unicode
(
xml_data
[
4
:
]
,
'utf-32le'
)
.
encode
(
'utf-8'
)
elif
xml_data
[
:
3
]
==
'\xef\xbb\xbf'
:
# UTF-8 with BOM
sniffed_xml_encoding
=
'utf-8'
xml_data
=
unicode
(
xml_data
[
3
:
]
,
'utf-8'
)
.
encode
(
'utf-8'
)
else
:
sniffed_xml_encoding
=
'ascii'
pass
except
:
xml_encoding_match
=
xml_encoding_match
=
re
.
compile
(
'^<\?.*encoding=[\'"](.*?)[\'"].*\?>'
)
.
match
(
xml_data
)
if
not
xml_encoding_match
and
isHTML
:
regexp
=
re
.
compile
(
'<\s*meta[^>]+charset=([^>]*?)[;\'">]'
,
re
.
I
)
xml_encoding_match
=
regexp
.
search
(
xml_data
)
if
xml_encoding_match
is
not
:
xml_encoding
=
xml_encoding_match
.
groups
(
)
[
0
]
.
lower
(
)
if
isHTML
:
self
.
declaredHTMLEncoding
=
xml_encoding
if
sniffed_xml_encoding
and
(
xml_encoding
in
(
'iso-10646-ucs-2'
,
'ucs-2'
,
'csunicode'
,
'iso-10646-ucs-4'
,
'ucs-4'
,
'csucs4'
,
'utf-16'
,
'utf-32'
,
'utf_16'
,
'utf_32'
,
'utf16'
,
'u16'
)
)
:
xml_encoding
=
sniffed_xml_encoding
return
xml_data
,
xml_encoding
,
sniffed_xml_encoding


def _in_tag(text, tag):
    """Extracts text from inside a tag.

    This function extracts the text from inside a given tag.
    It's useful to get the text between  or

 when using the validators or the colorizer.
    """
    if text.count('<%s' % tag):
        text = text.split('<%s' % tag, 1)[1]
        if text.count('>'):
            text = text.split('>', 1)[1]
    if text.count('</%s' % tag):
        text = text.split('</%s' % tag, 1)[0]

    text = text.strip().replace('\r\n', '\n')

    return text

def _in_tag ( text , tag ) : if text . count ( '<%s' % tag ) : text = text . split ( '<%s' % tag , 1 ) [ 1 ] if text . count ( '>' ) : text = text . split ( '>' , 1 ) [ 1 ] if text . count ( '</%s' % tag ) : text = text . split ( '</%s' % tag , 1 ) [ 0 ] text = text . strip ( ) . replace ( '\r\n' , '\n' ) return text


Outputs debug information to sys . stderr .

def _debug(s, level=1):
    """Outputs debug information to sys.stderr.

    This function outputs debug information if DEBUGLEVEL is
    higher than a given treshold.
    """
    if DEBUGLEVEL >= level: print >> sys.stderr, s

def _debug ( s , level = 1 ) : if DEBUGLEVEL >= level : print >> sys . stderr , s


Alternative re . sub that handles empty groups .

def preg_replace(pattern, replacement, text):
    """Alternative re.sub that handles empty groups.

    This acts like re.sub, except it replaces empty groups with ''
    instead of raising an exception.
    """

    def replacement_func(matchobj):
        counter = 1
        rc = replacement
        _debug(matchobj.groups())
        for matchitem in matchobj.groups():
            if not matchitem:
                matchitem = ''

            rc = rc.replace(r'\%s' % counter, matchitem)
            counter += 1

        return rc

    p = re.compile(pattern)
    _debug(pattern)

    return p.sub(replacement_func, text)

def preg_replace ( pattern , replacement , text ) : def replacement_func ( matchobj ) : counter = 1 rc = replacement _debug ( matchobj . groups ( ) ) for matchitem in matchobj . groups ( ) : if not matchitem : matchitem = '' rc = rc . replace ( r'\%s' % counter , matchitem ) counter += 1 return rc p = re . compile ( pattern ) _debug ( pattern ) return p . sub ( replacement_func , text )


def html_replace(pattern, replacement, text):
    """Replacement outside HTML tags.

    Does a preg_replace only outside HTML tags.
    """
    # If there is no html, do a simple search and replace.
    if not re.search(r'''<.*>''', text):
        return preg_replace(pattern, replacement, text)

    else:
        lines = []
        # Else split the text into an array at <>.
        for line in re.split('(<.*?>)', text):
            if not re.match('<.*?>', line):
                line = preg_replace(pattern, replacement, line)

            lines.append(line)

        return ''.join(lines)


def textile(text, **args):
    """This is Textile.

    Generates XHTML from a simple markup developed by Dean Allen.

    This function should be called like this:

        textile(text, head_offset=0, validate=0, sanitize=0,
                encoding='latin-1', output='ASCII')
    """
    return Textiler(text).process(**args)

def textile ( text , * * args ) : return Textiler ( text ) . process ( * * args )


def output(self):
        """Return processed HTML as a single string"""
        return "".join(self.pieces)

def output ( self ) : return "" . join ( self . pieces )


Instantiate the class passing the text to be formatted . Here we pre - process the text and collect all the link lookups for later .

def __init__(self, text=''):
        """Instantiate the class, passing the text to be formatted.

        Here we pre-process the text and collect all the link
        lookups for later.
        """
        self.text = text

        # Basic regular expressions.
        self.res = res

        # Smart searches.
        self.searches = {}
        self.searches['imdb']   = 'http://www.imdb.com/Find?for=%s'
        self.searches['google'] = 'http://www.google.com/search?q=%s'
        self.searches['python'] = 'http://www.python.org/doc/current/lib/module-%s.html'
        if amazon_associate_id:
            self.searches['isbn']   = ''.join(['http://', AMAZON, '/exec/obidos/ASIN/%s/', amazon_associate_id])
            self.searches['amazon'] = ''.join(['http://', AMAZON, '/exec/obidos/external-search?mode=blended&keyword=%s&tag=', amazon_associate_id])
        else:
            self.searches['isbn']   = ''.join(['http://', AMAZON, '/exec/obidos/ASIN/%s'])
            self.searches['amazon'] = ''.join(['http://', AMAZON, '/exec/obidos/external-search?mode=blended&keyword=%s'])

        # These are the blocks we know.
        self.signatures = [
                           # Paragraph.
                           (r'''^p                       # Paragraph signature
                                %(battr)s                # Paragraph attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended paragraph denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.paragraph),

                           # Pre-formatted text.
                           (r'''^pre                     # Pre signature
                                %(battr)s                # Pre attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended pre denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.pre),

                           # Block code.
                           (r'''^bc                      # Blockcode signature
                                %(battr)s                # Blockcode attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended blockcode denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.bc),

                           # Blockquote.
                           (r'''^bq                      # Blockquote signature
                                %(battr)s                # Blockquote attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended blockquote denoted by a second dot
                                (:(?P<cite>              # Optional cite attribute
                                (                        #
                                    %(url)s              #     URL
                                |   "[\w]+(?:\s[\w]+)*"  #     "Name inside quotes"
                                ))                       #
                                )?                       #
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.blockquote),

                           # Header.
                           (r'''^h                       # Header signature
                                (?P<header>\d)           # Header number
                                %(battr)s                # Header attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended header denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.header),

                           # Footnote.
                           (r'''^fn                      # Footnote signature
                                (?P<footnote>[\d]+)      # Footnote number
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended footnote denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''', self.footnote),

                           # Definition list.
                           (r'''^dl                      # Definition list signature
                                %(battr)s                # Definition list attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended definition list denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.dl),

                           # Ordered list (attributes to first <li>).
                           (r'''^%(olattr)s              # Ordered list attributes
                                \#                       # Ordered list signature
                                %(liattr)s               # List item attributes
                                (?P<dot>\.)?             # .
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.ol),

                           # Unordered list (attributes to first <li>).
                           (r'''^%(olattr)s              # Unrdered list attributes
                                \*                       # Unordered list signature
                                %(liattr)s               # Unordered list attributes
                                (?P<dot>\.)?             # .
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.ul),

                           # Escaped text.
                           (r'''^==?(?P<text>.*?)(==)?$  # Escaped text
                             ''', self.escape),

                           (r'''^(?P<text><.*)$          # XHTML tag
                             ''', self.escape),

                           # itex code.
                           (r'''^(?P<text>               # itex code
                                \\\[                     # starts with \[
                                .*?                      # complicated mathematical equations go here
                                \\\])                    # ends with \]
                             ''', self.itex),

                           # Tables.
                           (r'''^table                   # Table signature
                                %(tattr)s                # Table attributes
                                (?P<dot>\.)              # .
                                (?P<extend>\.)?          # Extended blockcode denoted by a second dot
                                \s                       # whitespace
                                (?P<text>.*)             # text
                             ''' % self.res, self.table),

                           # Simple tables.
                           (r'''^(?P<text>
                                \|
                                .*)
                             ''', self.table),

                           # About.
                           (r'''^(?P<text>tell\sme\sabout\stextile\.)$''', self.about),
                          ]


def preprocess(self):
        """Pre-processing of the text.

        Remove whitespace, fix carriage returns.
        """
        # Remove whitespace.
        self.text = self.text.strip()

        # Zap carriage returns.
        self.text = self.text.replace("\r\n", "\n")
        self.text = self.text.replace("\r", "\n")

        # Minor sanitizing.
        self.text = self.sanitize(self.text)

def
preprocess
(
self
)
:
# Remove whitespace.
self
.
text
=
self
.
text
.
strip
(
)
# Zap carriage returns.
self
.
text
=
self
.
text
.
replace
(
"\r\n"
,
"\n"
)
self
.
text
=
self
.
text
.
replace
(
"\r"
,
"\n"
)
# Minor sanitizing.
self
.
text
=
self
.
sanitize
(
self
.
text
)


def grab_links(self):
        """Grab link lookups.

        Check the text for link lookups, store them in a
        dictionary, and clean them up.
        """
        # Grab links like this: '[id]example.com'
        links = {}
        p = re.compile(r'''(?:^|\n)\[([\w]+?)\](%(url)s)(?:$|\n)''' % self.res, re.VERBOSE)
        for key, link in p.findall(self.text):
            links[key] = link

        # And clear them from the text.
        self.text = p.sub('', self.text)

        return links


def process(self, head_offset=HEAD_OFFSET, validate=VALIDATE, sanitize=SANITIZE, output=OUTPUT, encoding=ENCODING):
        """Process the text.

        Here we actually process the text, splitting the text in
        blocks and applying the corresponding function to each
        one of them.
        """
        # Basic global changes.
        self.preprocess()

        # Grab lookup links and clean them from the text.
        self._links = self.grab_links()

        # Offset for the headers.
        self.head_offset = head_offset

        # Process each block.
        self.blocks = self.split_text()

        text = []
        for [function, captures] in self.blocks:
            text.append(function(**captures))

        text = '\n\n'.join(text)

        # Add titles to footnotes.
        text = self.footnotes(text)

        # Convert to desired output.
        text = unicode(text, encoding)
        text = text.encode(output, 'xmlcharrefreplace')

        # Sanitize?
        if sanitize:
            p = _HTMLSanitizer()
            p.feed(text)
            text = p.output()

        # Validate output.
        if _tidy and validate:
            text = _tidy(text)

        return text


def sanitize(self, text):
        """Fix single tags.

        Fix tags like ,
 and
.

        ---
        h1. Sanitizing

        Textile can help you generate valid XHTML(eXtensible HyperText Markup Language).
        It will fix any single tags that are not properly closed, like
        @@, @
@ and @
@.

        If you have "mx.Tidy":http://www.egenix.com/files/python/mxTidy.html
        and/or "µTidyLib":http://utidylib.sourceforge.net/ installed,
        it also can optionally validade the generated code with these wrappers
        to ensure 100% valid XHTML(eXtensible HyperText Markup Language).
        """
        # Fix single tags like  and
.
        text = preg_replace(r'''<(img|br|hr)(.*?)(?:\s*/?\s*)?>''', r'''<\1\2 />''', text)

        # Remove ampersands.
        text = preg_replace(r'''&(?!#?[xX]?(?:[0-9a-fA-F]+|\w{1,8});)''', r'''&''', text)

        return text

# Fix single tags like <img /> and <br />.


def split_text(self):
        """Process the blocks from the text.

        Split the blocks according to the signatures, join extended
        blocks and associate each one of them with a function to
        process them.

        ---
        h1. Blocks

        Textile process your text by dividing it in blocks. Each block
        is identified by a signature and separated from other blocks by
        an empty line.

        All signatures should end with a period followed by a space. A
        header @

@ can be done this way:

        pre. h1. This is a header 1.

        Blocks may continue for multiple paragraphs of text. If you want
        a block signature to stay "active", use two periods after the
        signature instead of one. For example:

        pre.. bq.. This is paragraph one of a block quote.

        This is paragraph two of a block quote.

        =p. Now we're back to a regular paragraph.

        p. Becomes:

        pre..

This is paragraph one of a block quote.

This is paragraph two of a block quote.

Now we’re back to a regular paragraph.

        p. The blocks can be customised by adding parameters between the
        signature and the period. These include:

        dl. {style rule}:A CSS(Cascading Style Sheets) style rule.
        [ll]:A language identifier (for a "lang" attribute).
        (class) or (#id) or (class#id):For CSS(Cascading Style Sheets) class and id attributes.
        >, <, =, <>:Modifier characters for alignment. Right-justification, left-justification, centered, and full-justification. The paragraph will also receive the class names "right", "left", "center" and "justify", respectively.
        ( (one or more):Adds padding on the left. 1em per "(" character is applied. When combined with the align-left or align-right modifier, it makes the block float.
        ) (one or more):Adds padding on the right. 1em per ")" character is applied. When combined with the align-left or align-right modifier, it makes the block float.

        Here's an overloaded example:

        pre. p(())>(class#id)[en]{color:red}. A simple paragraph.

        Becomes:

        pre.
A simple paragraph.

        """
        # Clear signature.
        clear_sig = r'''^clear(?P<alignment>[<>])?\.$'''
        clear = None

        extending  = 0

        # We capture the \n's because they are important inside "pre..".
        blocks = re.split(r'''((\n\s*){2,})''', self.text)
        output = []
        for block in blocks:
            # Check for the clear signature.
            m = re.match(clear_sig, block)
            if m:
                clear = m.group('alignment')
                if clear:
                    clear = {'<': 'clear:left;', '>': 'clear:right;'}[clear]
                else:
                    clear = 'clear:both;'

            else:
                # Check each of the code signatures.
                for regexp, function in self.signatures:
                    p = re.compile(regexp, (re.VERBOSE | re.DOTALL))
                    m = p.match(block)
                    if m:
                        # Put everything in a dictionary.
                        captures = m.groupdict()

                        # If we are extending a block, we require a dot to
                        # break it, so we can start lines with '#' inside
                        # an extended
 without matching an ordered list.
                        if extending and not captures.get('dot', None):
                            output[-1][1]['text'] += block
                            break
                        elif captures.has_key('dot'):
                            del captures['dot']

                        # If a signature matches, we are not extending a block.
                        extending = 0

                        # Check if we should extend this block.
                        if captures.has_key('extend'):
                            extending = captures['extend']
                            del captures['extend']

                        # Apply head_offset.
                        if captures.has_key('header'):
                            captures['header'] = int(captures['header']) + self.head_offset

                        # Apply clear.
                        if clear:
                            captures['clear'] = clear
                            clear = None

                        # Save the block to be processed later.
                        output.append([function, captures])

                        break

                else:
                    if extending:
                        # Append the text to the last block.
                        output[-1][1]['text'] += block
                    elif block.strip():
                        output.append([self.paragraph, {'text': block}])

        return output

# an extended <pre> without matching an ordered list.