#NEWLINE# def solve( self ) : #NEWLINE# #INDENT# 'Solve the domain MDP.' #NEWLINE# self . start_time = clock ( ) #NEWLINE# bellmanUpdates = 0 #NEWLINE# converged = False #NEWLINE# iteration = 0 #NEWLINE# converged_trajectories = 0 #NEWLINE# while ( self . hasTime ( ) and ( not converged ) ) : #NEWLINE# #INDENT# max_Bellman_Error = 0 #NEWLINE# step = 0 #NEWLINE# terminal = False #NEWLINE# ( s , terminal , p_actions ) = self . domain . s0 ( ) #NEWLINE# a = ( self . representation . bestAction ( s , terminal , p_actions ) if ( np . random . rand ( ) > self . epsilon ) else randSet ( p_actions ) ) #NEWLINE# while ( ( not terminal ) and ( step < self . domain . episodeCap ) and self . hasTime ( ) ) : #NEWLINE# #INDENT# new_Q = self . representation . Q_oneStepLookAhead ( s , a , self . ns_samples ) #NEWLINE# phi_s = self . representation . phi ( s , terminal ) #NEWLINE# phi_s_a = self . representation . phi_sa ( s , terminal , a , phi_s ) #NEWLINE# old_Q = np . dot ( phi_s_a , self . representation . weight_vec ) #NEWLINE# bellman_error = ( new_Q - old_Q ) #NEWLINE# self . representation . weight_vec += ( ( self . alpha * bellman_error ) * phi_s_a ) #NEWLINE# bellmanUpdates += 1 #NEWLINE# step += 1 #NEWLINE# discover_func = getattr ( self . representation , 'discover' , None ) #NEWLINE# if ( discover_func and callable ( discover_func ) ) : #NEWLINE# #INDENT# self . representation . discover ( phi_s , bellman_error ) #NEWLINE# #UNINDENT# max_Bellman_Error = max ( max_Bellman_Error , abs ( bellman_error ) ) #NEWLINE# ( _ , s , terminal , p_actions ) = self . domain . step ( a ) #NEWLINE# a = ( self . representation . bestAction ( s , terminal , p_actions ) if ( np . random . rand ( ) > self . epsilon ) else randSet ( p_actions ) ) #NEWLINE# #UNINDENT# iteration += 1 #NEWLINE# if ( max_Bellman_Error < self . convergence_threshold ) : #NEWLINE# #INDENT# converged_trajectories += 1 #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# converged_trajectories = 0 #NEWLINE# #UNINDENT# ( performance_return , performance_steps , performance_term , performance_discounted_return ) = self . performanceRun ( ) #NEWLINE# converged = ( converged_trajectories >= self . MIN_CONVERGED_TRAJECTORIES ) #NEWLINE# self . logger . info ( ( 'PI #%d [%s]: BellmanUpdates=%d, ||Bellman_Error||=%0.4f, Return=%0.4f, Steps=%d, Features=%d' % ( iteration , hhmmss ( deltaT ( self . start_time ) ) , bellmanUpdates , max_Bellman_Error , performance_return , performance_steps , self . representation . features_num ) ) ) #NEWLINE# if self . show : #NEWLINE# #INDENT# self . domain . show ( a , representation = self . representation , s = s ) #NEWLINE# #UNINDENT# self . result [ 'bellman_updates' ] . append ( bellmanUpdates ) #NEWLINE# self . result [ 'return' ] . append ( performance_return ) #NEWLINE# self . result [ 'planning_time' ] . append ( deltaT ( self . start_time ) ) #NEWLINE# self . result [ 'num_features' ] . append ( self . representation . features_num ) #NEWLINE# self . result [ 'steps' ] . append ( converged_trajectories ) #NEWLINE# self . result [ 'terminated' ] . append ( performance_term ) #NEWLINE# self . result [ 'discounted_return' ] . append ( performance_discounted_return ) #NEWLINE# self . result [ 'iteration' ] . append ( iteration ) #NEWLINE# #UNINDENT# if converged : #NEWLINE# #INDENT# self . logger . info ( 'Converged!' ) #NEWLINE# #UNINDENT# super ( TrajectoryBasedValueIteration , self ) . solve ( )