#NEWLINE# def filter_extremes( self , no_below = 5 , no_above = 0.5 , keep_n = 100000 ) : #NEWLINE# #INDENT# '\n        Remove document frequency statistics for tokens that appear in\n\n        1. less than `no_below` documents (absolute number) or\n        2. more than `no_above` documents (fraction of total corpus size, *not*\n           absolute number).\n        3. after (1) and (2), keep only the first `keep_n` most frequent tokens (or\n           keep all if `None`).\n\n        **Note:** since HashDictionary\'s id range is fixed and doesn\'t depend on\n        the number of tokens seen, this doesn\'t really "remove" anything. It only\n        clears some supplementary statistics, for easier debugging and a smaller RAM\n        footprint.\n        ' #NEWLINE# no_above_abs = int ( ( no_above * self . num_docs ) ) #NEWLINE# ok = [ item for item in iteritems ( self . dfs_debug ) if ( no_below <= item [ 1 ] <= no_above_abs ) ] #NEWLINE# ok = frozenset ( ( word for ( word , freq ) in sorted ( ok , key = ( lambda item : ( - item [ 1 ] ) ) ) [ : keep_n ] ) ) #NEWLINE# self . dfs_debug = dict ( ( ( word , freq ) for ( word , freq ) in iteritems ( self . dfs_debug ) if ( word in ok ) ) ) #NEWLINE# self . token2id = dict ( ( ( token , tokenid ) for ( token , tokenid ) in iteritems ( self . token2id ) if ( token in self . dfs_debug ) ) ) #NEWLINE# self . id2token = dict ( ( ( tokenid , set ( ( token for token in tokens if ( token in self . dfs_debug ) ) ) ) for ( tokenid , tokens ) in iteritems ( no_below . id2token ) ) ) #NEWLINE# self . dfs = dict ( ( ( tokenid , freq ) for ( tokenid , freq ) in iteritems ( self . dfs ) if self . id2token . get ( tokenid , set ( ) ) ) ) #NEWLINE# logger . info ( ( 'kept statistics for which were in no less than %i and no more than %i (=%.1f%%) documents' % ( no_below , no_above_abs , ( 100.0 * no_above ) ) ) )