#NEWLINE# def load_dictionary( self ) : #NEWLINE# #INDENT# 'Read a file with a word list and create a dictionary.' #NEWLINE# logger = logging . getLogger ( 'Logger' ) #NEWLINE# logger . info ( 'Loading vocabulary' ) #NEWLINE# key = ( 'vocabulary_%s' % self . task ) #NEWLINE# filename = self . md . paths [ key ] #NEWLINE# if ( not os . path . isfile ( filename ) ) : #NEWLINE# #INDENT# filename = self . md . paths [ 'vocabulary' ] #NEWLINE# if ( not os . path . isfile ( filename ) ) : #NEWLINE# #INDENT# raise FileNotFoundException ( ) #NEWLINE# #UNINDENT# #UNINDENT# words = [ ] #NEWLINE# with open ( filename , 'rb' ) as f : #NEWLINE# #INDENT# for word in f : #NEWLINE# #INDENT# word = unicode ( word , 'utf-8' ) . strip ( ) #NEWLINE# if word : #NEWLINE# #INDENT# words . append ( word ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# wd = WordDictionary . init_from_wordlist ( words ) #NEWLINE# self . word_dict = wd #NEWLINE# logger . debug ( ( 'Done. Dictionary size is %d types' % wd . num_tokens ) )