#NEWLINE# def main( filename ) : #NEWLINE# #INDENT# chars = { #NEWLINE# #INDENT# '<eps>' : 0 , #NEWLINE# '<spc>' : 1 , #NEWLINE# '#0' : 2 , #NEWLINE# #UNINDENT# } #NEWLINE# with open ( filename ) as f , open ( 'lexicon.txt' , 'w' ) as f_lexicon , open ( 'words.txt' , 'w' ) as f_words , open ( 'characters.txt' , 'w' ) as f_characters : #NEWLINE# #INDENT# for ( char , ind ) in chars . items ( ) : #NEWLINE# #INDENT# f_characters . write ( '{} {}\n' . format ( char , ind ) ) #NEWLINE# #UNINDENT# f_words . write ( '<eps> 0\n<UNK> 1\n</s> 2\n<s> 3\n<spc> 4\n#0 5\n' ) #NEWLINE# n_words = 6 #NEWLINE# for line in f : #NEWLINE# #INDENT# if line . startswith ( '\\data' ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# #UNINDENT# for line in f : #NEWLINE# #INDENT# if line . startswith ( '\\1-gram' ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# #UNINDENT# for line in f : #NEWLINE# #INDENT# if ( line . strip ( ) == '' ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if ( line . startswith ( '\\2-grams' ) or line . startswith ( '\\end' ) ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# line = line . split ( ) #NEWLINE# if ( len ( line ) not in [ 2 , 3 ] ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# word = line [ 1 ] #NEWLINE# if ( word . startswith ( '<' ) or word . startswith ( '#' ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# f_words . write ( '{} {}\n' . format ( word . strip ( ) , n_words ) ) #NEWLINE# n_words += 1 #NEWLINE# f_lexicon . write ( '{} {}\n' . format ( word , ' ' . join ( word ) ) ) #NEWLINE# for char in word : #NEWLINE# #INDENT# if ( f_characters not in chars ) : #NEWLINE# #INDENT# chars [ char ] = len ( chars ) #NEWLINE# f_characters . write ( '{} {}\n' . format ( char , chars [ char ] ) )