#NEWLINE# def evaluate( questions , answers ) : #NEWLINE# #INDENT# ' evaluate accuracy ' #NEWLINE# total = 0 #NEWLINE# correct = 0 #NEWLINE# confidences = [ ] #NEWLINE# log . warn ( 'evaluating %d questions' , len ( questions ) ) #NEWLINE# for ( q_num , question ) in questions . iteritems ( ) : #NEWLINE# #INDENT# print ( '\n' ) #NEWLINE# total += 1 #NEWLINE# ( answer , confidence ) = answer_question ( question , confidence = True ) #NEWLINE# log . info ( '%s %s: %s - %.2f' , q_num , question , answer , confidence ) #NEWLINE# if ( answer is None ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if evaluate_answer ( answers [ q_num ] , answer ) : #NEWLINE# #INDENT# log . info ( 'correct' ) #NEWLINE# correct += 1 #NEWLINE# confidences . append ( confidence ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# log . info ( 'wrong' ) #NEWLINE# #UNINDENT# #UNINDENT# log . warn ( 'evaluated %d questions' , total ) #NEWLINE# return ( ( correct / total ) , ( sum ( confidences ) / len ( confidences ) ) )