#NEWLINE# def create_ngram_prob( lang , db = ':memory:' ) : #NEWLINE# #INDENT# table_name = 'lang{}trigram_without_last' . format ( lang ) #NEWLINE# con = sqlite3 . connect ( db ) #NEWLINE# cur = con . cursor ( ) #NEWLINE# trigram_tablename = 'lang{}trigram' . format ( lang ) #NEWLINE# trigramprob_tablename = 'lang{}trigramprob' . format ( lang ) #NEWLINE# trigramprobwithoutlast_tablename = 'lang{}trigramprob_without_last' . format ( lang ) #NEWLINE# Trigram = Tables ( ) . get_trigram_table ( trigram_tablename ) #NEWLINE# TrigramProb = Tables ( ) . get_trigramprob_table ( trigramprob_tablename ) #NEWLINE# TrigramProbWithoutLast = Tables ( ) . get_trigramprobwithoutlast_table ( trigramprobwithoutlast_tablename ) #NEWLINE# sqlalchemydb = 'sqlite:///{}' . format ( db ) #NEWLINE# engine = create_engine ( sqlalchemydb ) #NEWLINE# Session = sessionmaker ( bind = engine ) #NEWLINE# session = Session ( ) #NEWLINE# TrigramProb . __table__ . drop ( engine , checkfirst = True ) #NEWLINE# TrigramProb . __table__ . create ( engine ) #NEWLINE# TrigramProbWithoutLast . __table__ . drop ( engine , checkfirst = True ) #NEWLINE# TrigramProbWithoutLast . __table__ . create ( engine ) #NEWLINE# query = session . query ( Trigram ) #NEWLINE# totalnumber = len ( query . all ( ) ) #NEWLINE# query = session . query ( Trigram ) #NEWLINE# for item in trigramprob : #NEWLINE# #INDENT# ( first , second , third ) = ( item . first , item . second , item . third ) #NEWLINE# count = item . count #NEWLINE# cur . execute ( 'select * from {} where                     first=? and                    second=?' . format ( table_name ) , ( first , second ) ) #NEWLINE# one = cur . fetchone ( ) #NEWLINE# if ( not one ) : #NEWLINE# #INDENT# print ( 'not found correspont first and second' ) #NEWLINE# continue #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# alpha = 0.00017 #NEWLINE# c = count #NEWLINE# n = one [ 2 ] #NEWLINE# v = totalnumber #NEWLINE# logprob = math . log ( ( ( c + alpha ) / ( n + ( alpha * v ) ) ) ) #NEWLINE# print ( '{}, {}, {}:                  log({} + {} / {} + {} + {}) = {}' . format ( first , second , third , c , alpha , n , alpha , v , logprob ) ) #NEWLINE# trigramprob = TrigramProb ( first = first , second = second , third = third , prob = logprob ) #NEWLINE# session . add ( trigramprob ) #NEWLINE# logprobwithoutlast = math . log ( ( alpha / ( n + ( alpha * v ) ) ) ) #NEWLINE# print ( '{}, {}, {}:                  log({} / {} + {} + {}) = {}' . format ( first , second , third , alpha , n , alpha , v , logprobwithoutlast ) ) #NEWLINE# probwl = TrigramProbWithoutLast ( first = first , second = second , prob = logprobwithoutlast ) #NEWLINE# session . add ( probwl ) #NEWLINE# #UNINDENT# #UNINDENT# session . commit ( )