#NEWLINE# def _get_words_from_dataset( dataset ) : #NEWLINE# #INDENT# 'Return a set of all words in a dataset.\n\n    :param dataset: A list of tuples of the form ``(words, label)`` where\n        ``words`` is either a string of a list of tokens.\n    ' #NEWLINE# def tokenize( words ) : #NEWLINE# #INDENT# if isinstance ( words , basestring ) : #NEWLINE# #INDENT# return word_tokenize ( words , include_punc = False ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return words #NEWLINE# #UNINDENT# #UNINDENT# all_words = chain . from_iterable ( ( tokenize ( words ) for ( words , _ ) in dataset ) ) #NEWLINE# return set ( all_words )