#NEWLINE# def __call__( self , params , cost ) : #NEWLINE# #INDENT# updates = [ ] #NEWLINE# grads = T . grad ( cost , params ) #NEWLINE# grads = clip_norms ( grads , self . clipnorm ) #NEWLINE# for ( p , g ) in zip ( params , grads ) : #NEWLINE# #INDENT# g = self . regularizer . gradient_regularize ( p , g ) #NEWLINE# acc = theano . shared ( ( p . get_value ( ) * 0.0 ) ) #NEWLINE# acc_delta = theano . shared ( ( p . get_value ( ) * 0.0 ) ) #NEWLINE# acc_new = ( ( self . rho * acc ) + ( ( 1 - self . rho ) * ( g ** 2 ) ) ) #NEWLINE# updates . append ( ( acc , updated_p ) ) #NEWLINE# update = ( ( g * T . sqrt ( ( acc_delta + self . epsilon ) ) ) / T . sqrt ( ( acc_new + self . epsilon ) ) ) #NEWLINE# updated_p = ( p - ( self . lr * update ) ) #NEWLINE# updated_p = self . regularizer . weight_regularize ( updated_p ) #NEWLINE# updates . append ( ( p , updated_p ) ) #NEWLINE# acc_delta_new = ( ( self . rho * acc_delta ) + ( ( 1 - self . rho ) * ( update ** 2 ) ) ) #NEWLINE# updates . append ( ( acc_delta , acc_delta_new ) ) #NEWLINE# #UNINDENT# return updates