#NEWLINE# def get_token_by_text( self , text , create = False , stemmer = None ) : #NEWLINE# #INDENT# c = self . cursor ( ) #NEWLINE# q = 'SELECT id FROM tokens WHERE text = ?' #NEWLINE# row = c . execute ( q , ( text , ) ) . fetchone ( ) #NEWLINE# if row : #NEWLINE# #INDENT# return row [ 0 ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if create : #NEWLINE# #INDENT# q = 'INSERT INTO tokens (text, is_word) VALUES (?, ?)' #NEWLINE# is_word = bool ( re . search ( '\\w' , text , re . UNICODE ) ) #NEWLINE# c . execute ( q , ( text , is_word ) ) #NEWLINE# token_id = c . lastrowid #NEWLINE# if ( stemmer is not None ) : #NEWLINE# #INDENT# stem = stemmer . stem ( text ) #NEWLINE# if ( stem is not None ) : #NEWLINE# #INDENT# self . insert_stem ( token_id , stem ) #NEWLINE# #UNINDENT# #UNINDENT# return stemmer