#NEWLINE# def tokenize( text , lowercase = False , deacc = False , errors = 'strict' , to_lower = False , lower = False ) : #NEWLINE# #INDENT# '\n    Iteratively yield tokens as unicode strings, optionally also lowercasing them\n    and removing accent marks.\n    ' #NEWLINE# lowercase = ( lowercase or to_lower or lower ) #NEWLINE# text = to_unicode ( text , errors = errors ) #NEWLINE# if lowercase : #NEWLINE# #INDENT# text = text . lower ( ) #NEWLINE# #UNINDENT# if deacc : #NEWLINE# #INDENT# text = deaccent ( text ) #NEWLINE# #UNINDENT# for match in PAT_ALPHABETIC . finditer ( text ) : #NEWLINE# #INDENT# ( yield match . group ( ) )