#NEWLINE# def create_affix_list( self , prefix_or_suffix , max_size , min_occurrences ) : #NEWLINE# #INDENT# "\n        Handle the creation of suffix and prefix lists.\n        \n        Check if there exists an affix list in the data directory. If there isn't,\n        create a new one based on the training sentences.\n        \n        :param prefix_or_suffix: string 'prefix' or 'suffix'\n        " #NEWLINE# affix_type = prefix_or_suffix . lower ( ) #NEWLINE# assert ( ( affix_type == 'suffix' ) or ( affix_type == 'prefix' ) ) #NEWLINE# filename = self . md . paths [ ( '%ses' % affix_type ) ] #NEWLINE# if os . path . isfile ( filename ) : #NEWLINE# #INDENT# return #NEWLINE# #UNINDENT# logger = logging . getLogger ( 'Logger' ) #NEWLINE# affixes_all_lengths = [ ] #NEWLINE# types = { re . sub ( '\\d' , '9' , token . lower ( ) ) for sent in self . sentences for ( token , _ ) in sent } #NEWLINE# for length in range ( 1 , ( max_size + 1 ) ) : #NEWLINE# #INDENT# if ( affix_type == 'suffix' ) : #NEWLINE# #INDENT# c = Counter ( ( type_ [ ( - length ) : ] for type_ in types if ( len ( type_ ) > length ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# c = Counter ( ( type_ [ : length ] for type_ in types if ( len ( type_ ) > length ) ) ) #NEWLINE# #UNINDENT# affixes_this_length = [ affix for affix in c if ( c [ affix ] >= min_occurrences ) ] #NEWLINE# affixes_all_lengths . extend ( affixes_this_length ) #NEWLINE# #UNINDENT# logger . info ( ( 'Created a list of %d %ses.' % ( len ( affixes_all_lengths ) , affix_type ) ) ) #NEWLINE# text = '\n' . join ( affixes_all_lengths ) #NEWLINE# with open ( filename , 'wb' ) as f : #NEWLINE# #INDENT# f . write ( text . encode ( 'utf-8' ) )