#NEWLINE# def load_vocab_from_corpus( in_file , freq , max_vocab_size ) : #NEWLINE# #INDENT# '\n    Load vocab information from corpus, and maintain vocab mapping and freq mapping.\n    freq: frequency restriction can be provided, and rare words will be <unk>.\n    max_vocab_size: Or instead max size could be provided to cut off the vocab.\n\n    Return vocab information and basic corpus information.\n    ' #NEWLINE# f = codecs . open ( in_file , 'r' , 'utf-8' ) #NEWLINE# words = [ ] #NEWLINE# vocab_map = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# freq_map = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# vocab_size = 0 #NEWLINE# num_train_words = 0 #NEWLINE# num_lines = 0 #NEWLINE# for line in f : #NEWLINE# #INDENT# tokens = re . split ( '\\s+' , line . strip ( ) ) #NEWLINE# num_train_words += len ( tokens ) #NEWLINE# for token in tokens : #NEWLINE# #INDENT# if ( token not in vocab_map ) : #NEWLINE# #INDENT# words . append ( token ) #NEWLINE# vocab_map [ token ] = vocab_size #NEWLINE# freq_map [ token ] = 0 #NEWLINE# vocab_size += 1 #NEWLINE# #UNINDENT# freq_map [ token ] += 1 #NEWLINE# #UNINDENT# num_lines += 1 #NEWLINE# if ( ( num_lines % 100000 ) == 0 ) : #NEWLINE# #INDENT# sys . stderr . write ( ( ' (%d) ' % num_lines ) ) #NEWLINE# #UNINDENT# #UNINDENT# f . close ( ) #NEWLINE# if ( ( freq > 0 ) or ( max_vocab_size > 0 ) ) : #NEWLINE# #INDENT# ( words , vocab_map , freq_map , vocab_size ) = update_vocab ( words , vocab_map , freq_map , freq , freq_map ) #NEWLINE# #UNINDENT# return ( words , vocab_map , freq_map , vocab_size , num_train_words , num_lines )