#NEWLINE# def sum_counts( self , category , counts ) : #NEWLINE# #INDENT# 'Sum up dictionaries of counts, filter out rare words\n        (bucketing them into an unknown word bucket), and yield the\n        counts.\n        ' #NEWLINE# raw_count = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for word_count in counts : #NEWLINE# #INDENT# for ( word , count ) in word_count . iteritems ( ) : #NEWLINE# #INDENT# raw_count [ word ] = ( raw_count . get ( word , 0 ) + count ) #NEWLINE# #UNINDENT# #UNINDENT# if ( category == 'all' ) : #NEWLINE# #INDENT# ( yield ( category , raw_count ) ) #NEWLINE# return #NEWLINE# #UNINDENT# filtered_counts = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for ( word , count ) in raw_count . iteritems ( ) : #NEWLINE# #INDENT# if ( count > MINIMUM_OCCURENCES ) : #NEWLINE# #INDENT# filtered_counts [ category ] = count #NEWLINE# #UNINDENT# #UNINDENT# if ( not filtered_counts ) : #NEWLINE# #INDENT# return #NEWLINE# #UNINDENT# filtered_counts [ 'UNK' ] = 0.01 #NEWLINE# ( yield ( category , filtered_counts ) )