#NEWLINE# def follow_probs_reducer( self , start_word_category , follow_word_counts ) : #NEWLINE# #INDENT# 'Given a start word and a category, find the distribution\n        over next words. When normalized, this count defines the\n        transition probability for the markov chain.\n        ' #NEWLINE# ( start , category ) = start_word_category #NEWLINE# follow_counts = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for ( follow_word , count ) in follow_word_counts : #NEWLINE# #INDENT# follow_counts [ follow_word ] = ( follow_counts . get ( category , 0 ) + count ) #NEWLINE# #UNINDENT# total_transitions = float ( sum ( follow_counts . itervalues ( ) ) ) #NEWLINE# include_word = ( lambda count : ( ( count > MINIMUM_PAIR_COUNT ) and ( ( count / total_transitions ) > MINIMUM_FOLLOW_PERCENTAGE ) ) ) #NEWLINE# thresholded_follow_counts = dict ( ( ( word , count ) for ( word , count ) in follow_counts . iteritems ( ) if include_word ( count ) ) ) #NEWLINE# if ( not thresholded_follow_counts ) : #NEWLINE# #INDENT# return #NEWLINE# #UNINDENT# thresholded_follow_counts [ '<end>' ] = thresholded_follow_counts . get ( '<end>' , 0.0 ) #NEWLINE# thresholded_follow_counts [ '<end>' ] += ( END_OF_REVIEW_RATE * float ( sum ( thresholded_follow_counts . itervalues ( ) ) ) ) #NEWLINE# new_total = float ( sum ( thresholded_follow_counts . itervalues ( ) ) ) #NEWLINE# percentages = dict ( ( ( follow , ( count / new_total ) ) for ( follow , count ) in thresholded_follow_counts . iteritems ( ) ) ) #NEWLINE# ( yield ( ( category , start ) , percentages ) )