#NEWLINE# def _get_document_tokens( document ) : #NEWLINE# #INDENT# if isinstance ( w , basestring ) : #NEWLINE# #INDENT# tokens = set ( ( strip_punc ( w , all = False ) for w in word_tokenize ( document , include_punc = False ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# tokens = set ( ( strip_punc ( w , all = False ) for w in document ) ) #NEWLINE# #UNINDENT# return tokens