#NEWLINE# def __init__( self , tokens , size = None , minimum_occurrences = None , wordlist = None ) : #NEWLINE# #INDENT# '\n        Fills a dictionary (to be used for indexing) with the most\n        common words in the given text.\n        \n        :param tokens: Either a list of tokens or a list of lists of tokens \n            (each token represented as a string).\n        :param size: Maximum number of token indices \n            (not including paddings, rare, etc.).\n        :param minimum_occurrences: The minimum number of occurrences a token must \n            have in order to be included.\n        :param wordlist: Use this list of words to build the dictionary. Overrides tokens\n            if not None and ignores maximum size.\n        ' #NEWLINE# if ( wordlist is None ) : #NEWLINE# #INDENT# c = self . _get_frequency_count ( tokens ) #NEWLINE# if ( minimum_occurrences is None ) : #NEWLINE# #INDENT# minimum_occurrences = 1 #NEWLINE# #UNINDENT# words = [ key for ( key , number ) in c . most_common ( ) if ( ( number >= minimum_occurrences ) and key ) ] #NEWLINE# if ( ( size is not None ) and ( size < len ( words ) ) ) : #NEWLINE# #INDENT# words = words [ : size ] #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# words = [ word . lower ( ) . translate ( WordDictionary . number_transformation ) for word in wordlist ] #NEWLINE# values = ( [ None ] * len ( words ) ) #NEWLINE# words = OD ( zip ( words , values ) ) . keys ( ) #NEWLINE# #UNINDENT# if ( size is None ) : #NEWLINE# #INDENT# size = len ( words ) #NEWLINE# #UNINDENT# for ( word , num ) in itertools . izip ( words , xrange ( size ) ) : #NEWLINE# #INDENT# self [ word ] = num #NEWLINE# #UNINDENT# special_symbols = [ WordDictionary . rare . lower ( ) , WordDictionary . padding_left . lower ( ) , WordDictionary . padding_right . lower ( ) ] #NEWLINE# for symbol in special_symbols : #NEWLINE# #INDENT# if ( symbol not in words ) : #NEWLINE# #INDENT# self [ symbol ] = size #NEWLINE# size += 1 #NEWLINE# #UNINDENT# #UNINDENT# self . check ( )