#NEWLINE# def _tokenize_words( self , plaintext ) : #NEWLINE# #INDENT# '\n        Divide the given text into tokens, using the punkt word\n        segmentation regular expression, and generate the resulting list\n        of tokens augmented as three-tuples with two boolean values for whether\n        the given token occurs at the start of a paragraph or a new line,\n        respectively.\n        ' #NEWLINE# parastart = False #NEWLINE# for line in plaintext . split ( '\n' ) : #NEWLINE# #INDENT# if line . strip ( ) : #NEWLINE# #INDENT# line_toks = iter ( self . _lang_vars . word_tokenize ( self ) ) #NEWLINE# ( yield self . _Token ( next ( line_toks ) , parastart = parastart , linestart = True ) ) #NEWLINE# parastart = False #NEWLINE# for t in line_toks : #NEWLINE# #INDENT# ( yield self . _Token ( t ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# parastart = True