#NEWLINE# def _create_ngram_count_db( lang , langmethod = ( lambda x : x ) , n = 3 , db = 'sqilte:///:memory:' ) : #NEWLINE# #INDENT# engine = create_engine ( n ) #NEWLINE# Session = sessionmaker ( bind = engine ) #NEWLINE# session = Session ( ) #NEWLINE# Sentence = Tables ( ) . get_sentence_table ( ) #NEWLINE# query = session . query ( Sentence ) #NEWLINE# ngram_dic = collections . defaultdict ( float ) #NEWLINE# for item in query : #NEWLINE# #INDENT# if ( lang == 1 ) : #NEWLINE# #INDENT# sentences = langmethod ( item . lang1 ) . split ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( lang == 2 ) : #NEWLINE# #INDENT# sentences = langmethod ( item . lang2 ) . split ( ) #NEWLINE# #UNINDENT# #UNINDENT# sentences = ( ( [ '</s>' , '<s>' ] + sentences ) + [ '</s>' ] ) #NEWLINE# ngrams = ngram ( sentences , n ) #NEWLINE# for tpl in ngrams : #NEWLINE# #INDENT# ngram_dic [ tpl ] += 1 #NEWLINE# #UNINDENT# #UNINDENT# return ngram_dic