def get_html(self, raw_body, markup_type):
        """ Convert various Drupal formats to html """

        utf8_body = fix_string(raw_body)

        def repl(tmatch):
            if tmatch:   # Assume latin-1.  Will be converted by Bloog.
                return textile.textile(tmatch.group(1),
                                       encoding='latin-1', output='latin-1')

        # Because Drupal textile formatting allows use of [textile][/textile]
        # delimeters, remove them.
        if markup_type == 'textile':
            pattern = re.compile('\[textile\](.*)\[/textile\]',
                                 re.MULTILINE | re.IGNORECASE | re.DOTALL)
            body = re.sub(pattern, repl, utf8_body)
        elif markup_type == 'filtered html':
            body = re.sub('\n', '<br />', utf8_body)
        else:
            body = raw_body
        return body


Converts Model properties into various formats .

def to_dict(model_obj, attr_list, init_dict_func=None):
    """Converts Model properties into various formats.

    Supply a init_dict_func function that populates a
    dictionary with values.  In the case of db.Model, this
    would be something like _to_entity().  You may also
    designate more complex properties in attr_list, like
      "counter.count"
    Each object in the chain will be retrieved.  In the
    example above, the counter object will be retrieved
    from model_obj's properties.  And then counter.count
    will be retrieved.  The value returned will have a
    key set to the last name in the chain, e.g. 'count'
    in the above example.
    """
    values = {}
    init_dict_func(values)
    for token in attr_list:
        elems = token.split('.')
        value = getattr(model_obj, elems[0])
        for elem in elems[1:]:
            value = getattr(value, elem)
        values[elems[-1]] = value
    return values

def to_dict ( model_obj , attr_list , init_dict_func = None ) : values = { } init_dict_func ( values ) for token in attr_list : elems = token . split ( '.' ) value = getattr ( model_obj , elems [ 0 ] ) for elem in elems [ 1 : ] : value = getattr ( value , elem ) values [ elems [ - 1 ] ] = value return values


Replaces any datastore types in a dictionary with standard types . Passed - in entities are assumed to be dictionaries with values that can be at most a single list level . These transformations are made : datetime . datetime - > string db . Key - > key hash suitable for regenerating key users . User - > dict with nickname and email TODO -- GeoPt when needed

def replace_datastore_types(entity):
    """Replaces any datastore types in a dictionary with standard types.

    Passed-in entities are assumed to be dictionaries with values that
    can be at most a single list level.  These transformations are made:
      datetime.datetime      -> string
      db.Key                 -> key hash suitable for regenerating key
      users.User             -> dict with 'nickname' and 'email'
    TODO -- GeoPt when needed
    """
    def get_replacement(value):
        if isinstance(value, datetime.datetime):
            return value.strftime("%s %s" % (DATE_FORMAT, TIME_FORMAT))
        elif isinstance(value, datetime.date):
            return value.strftime(DATE_FORMAT)
        elif isinstance(value, datetime.time):
            return value.strftime(TIME_FORMAT)
        elif isinstance(value, datastore_types.Key):
            return str(value)
        elif isinstance(value, users.User):
            return { 'nickname': value.nickname(),
                     'email': value.email() }
        else:
            return None

    for key, value in entity.iteritems():
        if isinstance(value, list):
            new_list = []
            for item in value:
                new_value = get_replacement(item)
                new_list.append(new_value or item)
            entity[key] = new_list
        else:
            new_value = get_replacement(value)
            if new_value:
                entity[key] = new_value

def
replace_datastore_types
(
entity
)
:
def
get_replacement
(
value
)
:
if
isinstance
(
value
,
datetime
.
datetime
)
:
return
value
.
strftime
(
"%s %s"
%
(
DATE_FORMAT
,
TIME_FORMAT
)
)
elif
isinstance
(
value
,
datetime
.
date
)
:
return
value
.
strftime
(
DATE_FORMAT
)
elif
isinstance
(
value
,
datetime
.
time
)
:
return
value
.
strftime
(
TIME_FORMAT
)
elif
isinstance
(
value
,
datastore_types
.
Key
)
:
return
str
(
value
)
elif
isinstance
(
value
,
users
.
User
)
:
return
{
'nickname'
:
value
.
nickname
(
)
,
'email'
:
value
.
email
(
)
}
else
:
return
for
key
,
value
in
entity
.
iteritems
(
)
:
if
isinstance
(
value
,
list
)
:
new_list
=
[
]
for
item
in
value
:
new_value
=
get_replacement
(
item
)
new_list
.
append
(
new_value
or
item
)
entity
[
key
]
=
new_list
else
:
new_value
=
get_replacement
(
value
)
if
new_value
:
entity
[
key
]
=
new_value


Returns a list of up to 1000 dicts of model values . Unless nocache is set to True memcache will be checked first . Returns : List of dicts with each dict holding an entities property names and values .

def list(cls, nocache=False):
        """Returns a list of up to 1000 dicts of model values.
           Unless nocache is set to True, memcache will be checked first.
        Returns:
          List of dicts with each dict holding an entities property names
          and values.
        """
        list_repr = memcache.get(cls.memcache_key())
        if nocache or list_repr is None:
            q = db.Query(cls)
            objs = q.fetch(limit=1000)
            list_repr = '[' + ','.join([obj._to_repr() for obj in objs]) + ']'
            memcache.set(cls.memcache_key(), list_repr)
        return eval(list_repr)

def list ( cls , nocache = False ) : list_repr = memcache . get ( cls . memcache_key ( ) ) if nocache or list_repr is None : q = db . Query ( cls ) objs = q . fetch ( limit = 1000 ) list_repr = '[' + ',' . join ( [ obj . _to_repr ( ) for obj in objs ] ) + ']' memcache . set ( cls . memcache_key ( ) , list_repr ) return eval ( list_repr )


Constructor . May be called as a copy constructor .

def __init__(self, kind_or_entity, *args, **kwargs):
    """Constructor. May be called as a copy constructor.

    If kind_or_entity is a datastore.Entity, copies it into this Entity.
    datastore.Get() and Query() returns instances of datastore.Entity, so this
    is useful for converting them back to SearchableEntity so that they'll be
    indexed when they're stored back in the datastore.

    Otherwise, passes through the positional and keyword args to the
    datastore.Entity constructor.

    Args:
      kind_or_entity: string or datastore.Entity
    """
    if isinstance(kind_or_entity, datastore.Entity):
      self._Entity__key = kind_or_entity._Entity__key
      self.update(kind_or_entity)
    else:
      super(SearchableEntity, self).__init__(kind_or_entity, *args, **kwargs)

def __init__ ( self , kind_or_entity , * args , * * kwargs ) : if isinstance ( kind_or_entity , datastore . Entity ) : self . _Entity__key = kind_or_entity . _Entity__key self . update ( kind_or_entity ) else : super ( SearchableEntity , self ) . __init__ ( kind_or_entity , * args , * * kwargs )


Rebuilds the full text index then delegates to the superclass .

def _ToPb(self):
    """Rebuilds the full text index, then delegates to the superclass.

    Returns:
      entity_pb.Entity
    """
    if SearchableEntity._FULL_TEXT_INDEX_PROPERTY in self:
      del self[SearchableEntity._FULL_TEXT_INDEX_PROPERTY]

    index = set()
    searchable = lambda (name, value): name not in self.unsearchable_properties
    for (name, values) in filter(searchable, self.items()):
      if not isinstance(values, list):
        values = [values]
      if (isinstance(values[0], basestring) and
          not isinstance(values[0], datastore_types.Blob)):
        for value in values:
          index.update(SearchableEntity._FullTextIndex(value))

    index_list = list(index)
    if index_list:
      self[SearchableEntity._FULL_TEXT_INDEX_PROPERTY] = index_list

    return super(SearchableEntity, self)._ToPb()

def _ToPb ( self ) : if SearchableEntity . _FULL_TEXT_INDEX_PROPERTY in self : del self [ SearchableEntity . _FULL_TEXT_INDEX_PROPERTY ] index = set ( ) searchable = lambda ( name , value ) : name not in self . unsearchable_properties for ( name , values ) in filter ( searchable , self . items ( ) ) : if not isinstance ( values , list ) : values = [ values ] if ( isinstance ( values [ 0 ] , basestring ) and not isinstance ( values [ 0 ] , datastore_types . Blob ) ) : for value in values : index . update ( SearchableEntity . _FullTextIndex ( value ) ) index_list = list ( index ) if index_list : self [ SearchableEntity . _FULL_TEXT_INDEX_PROPERTY ] = index_list return super ( SearchableEntity , self ) . _ToPb ( )


Returns a set of keywords appropriate for full text indexing .

def _FullTextIndex(cls, text):
    """Returns a set of keywords appropriate for full text indexing.

    See SearchableQuery.Search() for details.

    Args:
      text: string

    Returns:
      set of strings
    """

    if text:
      datastore_types.ValidateString(text, 'text', max_len=sys.maxint)
      # TODO -- Remove embedded code blogs marked by 'pre' tags
      # and name="code"
      text = cls._PUNCTUATION_REGEX.sub(' ', text)
      words = text.lower().split()

      words = set(words)

      words -= cls._FULL_TEXT_STOP_WORDS
      for word in list(words):
        if len(word) < cls._FULL_TEXT_MIN_LENGTH:
          words.remove(word)

    else:
      words = set()

    return words


Add a search query . This may be combined with filters .

def Search(self, search_query):
    """Add a search query. This may be combined with filters.

    Note that keywords in the search query will be silently dropped if they
    are stop words or too short, ie if they wouldn't be indexed.

    Args:
     search_query: string

    Returns:
      # this query
      SearchableQuery
    """
    datastore_types.ValidateString(search_query, 'search query')
    self._search_query = search_query
    return self

def
Search
(
self
,
search_query
)
:
datastore_types
.
ValidateString
(
search_query
,
'search query'
)
self
.
_search_query
=
search_query
return
self


Adds filters for the search query then delegates to the superclass .

def _ToPb(self, limit=None, offset=None):
    """Adds filters for the search query, then delegates to the superclass.

    Raises BadFilterError if a filter on the index property already exists.

    Args:
      # an upper bound on the number of results returned by the query.
      limit: int
      # number of results that match the query to skip.  limit is applied
      # after the offset is fulfilled.
      offset: int

    Returns:
      datastore_pb.Query
    """
    if SearchableEntity._FULL_TEXT_INDEX_PROPERTY in self:
      raise datastore_errors.BadFilterError(
        '%s is a reserved name.' % SearchableEntity._FULL_TEXT_INDEX_PROPERTY)

    pb = super(SearchableQuery, self)._ToPb(limit=limit, offset=offset)

    if hasattr(self, '_search_query'):
      keywords = SearchableEntity._FullTextIndex(self._search_query)
      for keyword in keywords:
        filter = pb.add_filter()
        filter.set_op(datastore_pb.Query_Filter.EQUAL)
        prop = filter.add_property()
        prop.set_name(SearchableEntity._FULL_TEXT_INDEX_PROPERTY)
        prop.mutable_value().set_stringvalue(keyword)

    return pb

def
_ToPb
(
self
,
limit
=
,
offset
=
)
:
if
SearchableEntity
.
_FULL_TEXT_INDEX_PROPERTY
in
self
:
raise
datastore_errors
.
BadFilterError
(
'%s is a reserved name.'
%
SearchableEntity
.
_FULL_TEXT_INDEX_PROPERTY
)
pb
=
super
(
SearchableQuery
,
self
)
.
_ToPb
(
limit
=
limit
,
offset
=
offset
)
if
hasattr
(
self
,
'_search_query'
)
:
keywords
=
SearchableEntity
.
_FullTextIndex
(
self
.
_search_query
)
for
keyword
in
keywords
:
filter
=
pb
.
add_filter
(
)
filter
.
set_op
(
datastore_pb
.
Query_Filter
.
EQUAL
)
prop
=
filter
.
add_property
(
)
prop
.
set_name
(
SearchableEntity
.
_FULL_TEXT_INDEX_PROPERTY
)
prop
.
mutable_value
(
)
.
set_stringvalue
(
keyword
)
return
pb


def search(self, search_query):
    """Adds a full text search to this query.

    Args:
      search_query, a string containing the full text search query.

    Returns:
      self
    """
    self._search_query = search_query
    return self

def search ( self , search_query ) : self . _search_query = search_query return self


Wraps db . Query . _get_query () and injects SearchableQuery .

def _get_query(self):
    """Wraps db.Query._get_query() and injects SearchableQuery."""
    query = db.Query._get_query(self, _query_class=SearchableQuery)
    if self._search_query:
      query.Search(self._search_query)
    return query

def _get_query ( self ) : query = db . Query . _get_query ( self , _query_class = SearchableQuery ) if self . _search_query : query . Search ( self . _search_query ) return query


Wraps db . Model . _populate_internal_entity () and injects SearchableEntity .

def _populate_internal_entity(self):
    """Wraps db.Model._populate_internal_entity() and injects
    SearchableEntity."""
    entity = db.Model._populate_internal_entity(self,
                                            _entity_class=SearchableEntity)
    entity.unsearchable_properties = self.__class__.unsearchable_properties
    return entity

def _populate_internal_entity ( self ) : entity = db . Model . _populate_internal_entity ( self , _entity_class = SearchableEntity ) entity . unsearchable_properties = self . __class__ . unsearchable_properties return entity


Wraps db . Model . from_entity () and injects SearchableEntity .

def from_entity(cls, entity):
    """Wraps db.Model.from_entity() and injects SearchableEntity."""
    if not isinstance(entity, SearchableEntity):
      entity = SearchableEntity(entity)
    return super(SearchableModel, cls).from_entity(entity)

def from_entity ( cls , entity ) : if not isinstance ( entity , SearchableEntity ) : entity = SearchableEntity ( entity ) return super ( SearchableModel , cls ) . from_entity ( entity )


def all(cls):
    """Returns a FullTextQuery for this kind."""
    return FullTextQuery(cls)

def all ( cls ) : return FullTextQuery ( cls )


def get_comments(self):
        """Return comments lexicographically sorted on thread string"""
        q = db.GqlQuery("SELECT * FROM Comment " +
                        "WHERE article = :1 " +
                        "ORDER BY thread ASC", self.key())
        return [comment for comment in q]

def
get_comments
(
self
)
:
q
=
db
.
GqlQuery
(
"SELECT * FROM Comment "
+
"WHERE article = :1 "
+
"ORDER BY thread ASC"
,
self
.
key
(
)
)
return
[
comment
for
comment
in
q
]


Serialize data that we d like to store with this article . Examples include relevant ( per article ) links and associated Amazon items .

def set_associated_data(self, data):
        """
        Serialize data that we'd like to store with this article.
        Examples include relevant (per article) links and associated
        Amazon items.
        """
        import pickle
        self.assoc_dict = pickle.dumps(data)

def set_associated_data ( self , data ) : import pickle self . assoc_dict = pickle . dumps ( data )


def next_comment_thread_string(self):
        'Returns thread string for next comment for this article'
        return get_thread_string(self, '')

def next_comment_thread_string ( self ) : return get_thread_string ( self , '' )


Returns a string suitable for inclusion in Atom XML feed Internal html property should already have XHTML entities converted into unicode . However ampersands are valid ASCII and will cause issues with XML so reconvert ampersands to valid XML entities &amp ;

def to_atom_xml(self):
        """Returns a string suitable for inclusion in Atom XML feed

        Internal html property should already have XHTML entities
        converted into unicode.  However, ampersands are valid ASCII
        and will cause issues with XML, so reconvert ampersands to
        valid XML entities &
        """
        import re
        return re.sub('&(?!amp;)', '&', self.html)

def to_atom_xml ( self ) : import re return re . sub ( '&(?!amp;)' , '&' , self . html )


def next_child_thread_string(self):
        'Returns thread string for next child of this comment'
        return get_thread_string(self.article, self.thread + '.')

def next_child_thread_string ( self ) : return get_thread_string ( self . article , self . thread + '.' )


This maps request strings to values in a hash optionally run through a function with previous request values as parameters to the func . 1 ) key - > just read in the corresponding request value 2 ) tuple ( key func ) - > Read the request value for the string key and pass it through func 3 ) tuple ( key func additional keys ... ) - > Get the request values for the additional keys and pass them through func before setting the key s value with the output . If a key is not present in the request then we do not insert a key with None or empty string . The key is simply absent therefore allowing you to use the returned hash to initial a Model instance .

def get_sent_properties(request_func, propname_list):
    """
    This maps request strings to values in a hash, optionally run through
    a function with previous request values as parameters to the func.
    1) key -> just read in the corresponding request value
    2) tuple (key, func) -> Read the request value for the string key
        and pass it through func
    3) tuple (key, func, additional keys...) -> Get the request
        values for the additional keys and pass them through func
        before setting the key's value with the output.
    If a key is not present in the request, then we do not insert a key
    with None or empty string.  The key is simply absent, therefore allowing
    you to use the returned hash to initial a Model instance.
    """
    prop_hash = {}
    for item in propname_list:
        if isinstance(item, basestring):
            key = item
            value = request_func(item)
        elif isinstance(item, tuple):
            key = item[0]
            prop_func = item[1]
            if len(item) <= 2:
                value = prop_func(request_func(key))
            else:
                try:
                    addl_keys = map(prop_hash.get, item[2:])
                    value = prop_func(*addl_keys)
                except:
                    return None
        if value:
            prop_hash[key] = value
    return prop_hash

def get_sent_properties ( request_func , propname_list ) : prop_hash = { } for item in propname_list : if isinstance ( item , basestring ) : key = item value = request_func ( item ) elif isinstance ( item , tuple ) : key = item [ 0 ] prop_func = item [ 1 ] if len ( item ) <= 2 : value = prop_func ( request_func ( key ) ) else : try : addl_keys = map ( prop_hash . get , item [ 2 : ] ) value = prop_func ( * addl_keys ) except : return None if value : prop_hash [ key ] = value return prop_hash


A decorator to automatically re - route overloaded POSTs that specify the real HTTP method in a _method query string .

def methods_via_query_allowed(handler_method):
    """
    A decorator to automatically re-route overloaded POSTs
    that specify the real HTTP method in a _method query string.

    To use it, decorate your post method like this:

    import restful
    ...
    @restful.methods_via_query_allowed
    def post(self):
      pass

    The decorator will check for a _method query string or POST argument,
    and if present, will redirect to delete(), put(), etc.
    """
    def redirect_if_needed(self, *args, **kwargs):
        real_verb = self.request.get('_method', None)
        if not real_verb and 'X-HTTP-Method-Override' in self.request.environ:
            real_verb = self.request.environ['X-HTTP-Method-Override']
        if real_verb:
            logging.debug("Redirected from POST. Detected method override = %s", real_verb)
            method = real_verb.upper()
            if method == 'HEAD':
                self.head(*args, **kwargs)
            elif method == 'PUT':
                self.put(*args, **kwargs)
            elif method == 'DELETE':
                self.delete(*args, **kwargs)
            elif method == 'TRACE':
                self.trace(*args, **kwargs)
            elif method == 'OPTIONS':
                self.head(*args, **kwargs)
            # POST and GET included for completeness
            elif method == 'POST':
                self.post(*args, **kwargs)
            elif method == 'GET':
                self.get(*args, **kwargs)
            else:
                self.error(405)
        else:
            handler_method(self, *args, **kwargs)
    return redirect_if_needed


By using DELETE on / Article / Comment / Tag you can delete the first entity of the desired kind . This is useful for writing utilities like clear_datastore . py .

def delete(self, path):
        """
        By using DELETE on /Article, /Comment, /Tag, you can delete the first
         entity of the desired kind.
        This is useful for writing utilities like clear_datastore.py.
        """
        # TODO: Add DELETE for articles off root like blog entry DELETE.
        model_class = path.lower()
        logging.debug("ArticleHandler#delete on %s", path)

        def delete_entity(query):
            targets = query.fetch(limit=1)
            if len(targets) > 0:
                if hasattr(targets[0], 'title'):
                    title = targets[0].title
                elif hasattr(targets[0], 'name'):
                    title = targets[0].name
                else:
                    title = ''
                logging.debug('Deleting %s %s', model_class, title)
                targets[0].delete()
                self.response.out.write('Deleted ' + model_class + ' ' + title)
                view.invalidate_cache()
            else:
                self.response.set_status(204, 'No more ' + model_class + ' entities')

        if model_class == 'article':
            query = models.blog.Article.all()
            delete_entity(query)
        elif model_class == 'comment':
            query = models.blog.Comment.all()
            delete_entity(query)
        elif model_class == 'tag':
            query = models.blog.Tag.all()
            delete_entity(query)
        else:
            article = db.Query(models.blog.Article). \
                         filter('permalink =', path).get()
            for key in article.tag_keys:
                db.get(key).counter.decrement()
            article.delete()
            view.invalidate_cache()
            restful.send_successful_response(self, "/")


Add a blog entry . Since we are POSTing the server handles creation of the permalink url .

def post(self, year, month):
        """ Add a blog entry. Since we are POSTing, the server handles
            creation of the permalink url. """
        logging.debug("MonthHandler#post on date %s, %s", year, month)
        process_article_submission(handler=self, article_type='blog entry')

def
post
(
self
,
year
,
month
)
:
logging
.
debug
(
"MonthHandler#post on date %s, %s"
,
year
,
month
)
process_article_submission
(
handler
=
self
,
article_type
=
'blog entry'
)


Adds a global or updates it if it already exists .

def set_global(self, name, value):
    """Adds a global, or updates it if it already exists.

    Also removes the global from the list of unpicklable names.

    Args:
      name: the name of the global to remove
      value: any picklable value
    """
    blob = db.Blob(pickle.dumps(value))

    if name in self.global_names:
      index = self.global_names.index(name)
      self.globals[index] = blob
    else:
      self.global_names.append(db.Text(name))
      self.globals.append(blob)

    self.remove_unpicklable_name(name)

def set_global ( self , name , value ) : blob = db . Blob ( pickle . dumps ( value ) ) if name in self . global_names : index = self . global_names . index ( name ) self . globals [ index ] = blob else : self . global_names . append ( db . Text ( name ) ) self . globals . append ( blob ) self . remove_unpicklable_name ( name )


def remove_global(self, name):
    """Removes a global, if it exists.

    Args:
      name: string, the name of the global to remove
    """
    if name in self.global_names:
      index = self.global_names.index(name)
      del self.global_names[index]
      del self.globals[index]

def remove_global ( self , name ) : if name in self . global_names : index = self . global_names . index ( name ) del self . global_names [ index ] del self . globals [ index ]


Returns a dictionary view of the globals .

def globals_dict(self):
    """Returns a dictionary view of the globals.
    """
    return dict((name, pickle.loads(val))
                for name, val in zip(self.global_names, self.globals))

def globals_dict ( self ) : return dict ( ( name , pickle . loads ( val ) ) for name , val in zip ( self . global_names , self . globals ) )


Adds a statement and list of names to the unpicklables .

def add_unpicklable(self, statement, names):
    """Adds a statement and list of names to the unpicklables.

    Also removes the names from the globals.

    Args:
      statement: string, the statement that created new unpicklable global(s).
      names: list of strings; the names of the globals created by the statement.
    """
    self.unpicklables.append(db.Text(statement))

    for name in names:
      self.remove_global(name)
      if name not in self.unpicklable_names:
        self.unpicklable_names.append(db.Text(name))

def add_unpicklable ( self , statement , names ) : self . unpicklables . append ( db . Text ( statement ) ) for name in names : self . remove_global ( name ) if name not in self . unpicklable_names : self . unpicklable_names . append ( db . Text ( name ) )


Removes a name from the list of unpicklable names if it exists .

def remove_unpicklable_name(self, name):
    """Removes a name from the list of unpicklable names, if it exists.

    Args:
      name: string, the name of the unpicklable global to remove
    """
    if name in self.unpicklable_names:
      self.unpicklable_names.remove(name)

def remove_unpicklable_name ( self , name ) : if name in self . unpicklable_names : self . unpicklable_names . remove ( name )