#NEWLINE# def parse( self , corpus_filename , key ) : #NEWLINE# #INDENT# assert ( type ( corpus_filename ) == str ) , 'the filename must be a string' #NEWLINE# assert ( type ( key ) == str ) , 'the key must be a string' #NEWLINE# wst = WhitespaceTokenizer ( ) #NEWLINE# with codecs . open ( corpus_filename , encoding = 'utf8' ) as input : #NEWLINE# #INDENT# corpus = [ wst . tokenize ( l ) for l in input ] #NEWLINE# #UNINDENT# return { #NEWLINE# #INDENT# key : corpus , #NEWLINE# #UNINDENT# }