#NEWLINE# def tokenize( self , text , include_punc = True , nested = False ) : #NEWLINE# #INDENT# 'Return a list of word tokens.\n\n        :param text: string of text.\n        :param include_punc: (optional) whether to include punctuation as separate\n            tokens. Default to True.\n\n        ' #NEWLINE# self . tokens = [ w for w in ( self . word_tokenize ( s , include_punc ) for s in self . sent_tokenize ( w ) ) ] #NEWLINE# if nested : #NEWLINE# #INDENT# return self . tokens #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return list ( chain . from_iterable ( self . tokens ) )