#NEWLINE# def policyImprovement( self , policy ) : #NEWLINE# #INDENT# ' Given a policy improve it by taking the greedy action in each state based on the value function\n            Returns the new policy\n        ' #NEWLINE# policyChanges = 0 #NEWLINE# i = 0 #NEWLINE# while ( ( i < self . representation . agg_states_num ) and self . hasTime ( ) ) : #NEWLINE# #INDENT# s = self . representation . stateID2state ( i ) #NEWLINE# if ( ( not self . domain . isTerminal ( s ) ) and len ( self . domain . possibleActions ( s ) ) ) : #NEWLINE# #INDENT# for a in self . domain . possibleActions ( s ) : #NEWLINE# #INDENT# if ( not self . hasTime ( ) ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# self . BellmanBackup ( s , a , self . ns_samples , policy ) #NEWLINE# #UNINDENT# if ( policy . pi ( s , False , self . domain . possibleActions ( s = s ) ) != self . representation . bestAction ( s , False , self . domain . possibleActions ( s = s ) ) ) : #NEWLINE# #INDENT# policyChanges += 1 #NEWLINE# #UNINDENT# #UNINDENT# i += 1 #NEWLINE# #UNINDENT# policy . representation . weight_vec = self . representation . weight_vec . copy ( ) #NEWLINE# ( performance_return , performance_steps , performance_term , performance_discounted_return ) = self . performanceRun ( ) #NEWLINE# self . logger . info ( ( 'PI #%d [%s]: BellmanUpdates=%d, Policy Change=%d, Return=%0.4f, Steps=%d' % ( self . policy_improvement_iteration , hhmmss ( deltaT ( self . start_time ) ) , self . bellmanUpdates , policyChanges , performance_return , performance_steps ) ) ) #NEWLINE# self . result [ 'bellman_updates' ] . append ( self . bellmanUpdates ) #NEWLINE# self . result [ 'return' ] . append ( performance_return ) #NEWLINE# self . result [ 'planning_time' ] . append ( deltaT ( self . start_time ) ) #NEWLINE# self . result [ 'num_features' ] . append ( self . representation . features_num ) #NEWLINE# self . result [ 'steps' ] . append ( performance_steps ) #NEWLINE# self . result [ 'terminated' ] . append ( s ) #NEWLINE# self . result [ 'discounted_return' ] . append ( performance_discounted_return ) #NEWLINE# self . result [ 'policy_improvement_iteration' ] . append ( self . policy_improvement_iteration ) #NEWLINE# return ( policy , policyChanges )