#NEWLINE# def forward( self , state , action , reward , state_dash , episode_end ) : #NEWLINE# #INDENT# num_of_batch = state . shape [ 0 ] #NEWLINE# s = Variable ( state ) #NEWLINE# s_dash = Variable ( state_dash ) #NEWLINE# q = self . q_func ( s ) #NEWLINE# tmp = self . q_func_target ( s_dash ) #NEWLINE# if ( self . use_gpu >= 0 ) : #NEWLINE# #INDENT# tmp = list ( map ( np . max , tmp . data . get ( ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# tmp = list ( map ( np . max , tmp . data ) ) #NEWLINE# #UNINDENT# max_q_dash = np . asanyarray ( tmp , dtype = np . float32 ) #NEWLINE# if ( self . use_gpu >= 0 ) : #NEWLINE# #INDENT# target = np . asanyarray ( q . data . get ( ) , dtype = np . float32 ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# target = np . array ( q . data , dtype = np . float32 ) #NEWLINE# #UNINDENT# for i in xrange ( num_of_batch ) : #NEWLINE# #INDENT# if ( not episode_end [ i ] [ 0 ] ) : #NEWLINE# #INDENT# tmp_ = ( reward [ i ] + ( self . gamma * max_q_dash [ i ] ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# tmp_ = reward [ i ] #NEWLINE# #UNINDENT# action_index = self . action_to_index ( action [ i ] ) #NEWLINE# target [ ( i , action_index ) ] = tmp_ #NEWLINE# #UNINDENT# if ( self . use_gpu >= 0 ) : #NEWLINE# #INDENT# target = cuda . to_gpu ( target ) #NEWLINE# #UNINDENT# td = ( Variable ( target ) - q ) #NEWLINE# td_tmp = ( td . data + ( 1000.0 * ( abs ( td . data ) <= 1 ) ) ) #NEWLINE# td_clip = ( ( td * ( abs ( td . data ) <= 1 ) ) + ( ( td / abs ( td_tmp ) ) * ( abs ( td . data ) > 1 ) ) ) #NEWLINE# zero_val = np . zeros ( ( self . replay_size , self . num_of_actions ) , dtype = np . float32 ) #NEWLINE# if ( self . use_gpu >= 0 ) : #NEWLINE# #INDENT# zero_val = cuda . to_gpu ( zero_val ) #NEWLINE# #UNINDENT# zero_val = Variable ( zero_val ) #NEWLINE# loss = F . mean_squared_error ( td_clip , zero_val ) #NEWLINE# return ( loss , q )