#NEWLINE# def __init__( self , data , hp ) : #NEWLINE# #INDENT# super ( Draw_lstm1 , self ) . __init__ ( self . __class__ . __name__ , log_sigma_encoder , hp ) #NEWLINE# self . sample_steps = True #NEWLINE# self . n_h = 256 #NEWLINE# self . n_t = 12 #NEWLINE# self . n_zpt = 32 #NEWLINE# self . n_z = ( self . n_t * self . n_zpt ) #NEWLINE# self . gates = 4 #NEWLINE# self . params = Parameters ( ) #NEWLINE# n_x = self . data [ 'n_x' ] #NEWLINE# n_h = self . n_h #NEWLINE# n_z = self . n_z #NEWLINE# n_zpt = self . n_zpt #NEWLINE# n_t = self . n_t #NEWLINE# gates = self . gates #NEWLINE# scale = hp . init_scale #NEWLINE# if ( hp . load_model and os . path . isfile ( self . filename ) ) : #NEWLINE# #INDENT# self . params . load ( self . filename ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# with self . params : #NEWLINE# #INDENT# W1 = shared_normal ( ( ( ( n_x * 2 ) + n_h ) , ( n_h * gates ) ) , scale = scale ) #NEWLINE# W11 = shared_normal ( ( n_h , ( n_h * gates ) ) , scale = scale ) #NEWLINE# W2 = shared_normal ( ( n_h , n_zpt ) , scale = scale ) #NEWLINE# W3 = shared_normal ( ( n_h , n_zpt ) , scale = scale ) #NEWLINE# W4 = shared_normal ( ( n_zpt , ( n_h * gates ) ) , scale = scale ) #NEWLINE# W44 = shared_normal ( ( n_h , ( n_h * gates ) ) , scale = scale ) #NEWLINE# W5 = shared_normal ( ( n_h , n_x ) , scale = scale ) #NEWLINE# b1 = shared_zeros ( ( ( n_h * gates ) , ) ) #NEWLINE# b10_h = shared_zeros ( ( n_h , ) ) #NEWLINE# b10_c = shared_zeros ( ( n_h , ) ) #NEWLINE# b2 = shared_zeros ( ( n_zpt , ) ) #NEWLINE# b3 = shared_zeros ( ( n_zpt , ) ) #NEWLINE# b4 = shared_zeros ( ( ( n_h * gates ) , ) ) #NEWLINE# b40_h = shared_zeros ( ( n_h , ) ) #NEWLINE# b40_c = shared_zeros ( ( n_h , ) ) #NEWLINE# b5 = shared_zeros ( ( n_x , ) ) #NEWLINE# c0 = shared_zeros ( ( n_x , ) ) #NEWLINE# #UNINDENT# #UNINDENT# def lstm( X , h , c , W , U , b ) : #NEWLINE# #INDENT# g_on = ( ( T . dot ( X , W ) + T . dot ( h , U ) ) + b ) #NEWLINE# i_on = T . nnet . sigmoid ( g_on [ : , : n_h ] ) #NEWLINE# f_on = T . nnet . sigmoid ( g_on [ : , n_h : ( 2 * n_h ) ] ) #NEWLINE# o_on = T . nnet . sigmoid ( g_on [ : , ( 2 * n_h ) : ( 3 * n_h ) ] ) #NEWLINE# c = ( ( f_on * c ) + ( i_on * T . tanh ( g_on [ : , ( 3 * n_h ) : ] ) ) ) #NEWLINE# h = ( o_on * T . tanh ( c ) ) #NEWLINE# return ( h , c ) #NEWLINE# #UNINDENT# p = self . params #NEWLINE# x = self . X #NEWLINE# input_size = x . shape [ 0 ] #NEWLINE# c = p . c0 #NEWLINE# h_encoder_h = p . b10_h #NEWLINE# h_encoder_c = p . b10_c #NEWLINE# h_decoder_h = ( T . zeros ( ( input_size , p . b40_h . shape [ 0 ] ) ) + p . b40_h ) #NEWLINE# h_decoder_c = p . b40_c #NEWLINE# log_qpz = 0 #NEWLINE# eps = srnd . normal ( ( n_t , input_size , n_zpt ) , dtype = theano . config . floatX ) #NEWLINE# for t in xrange ( 0 , n_t ) : #NEWLINE# #INDENT# x_e = ( x - T . nnet . sigmoid ( c ) ) #NEWLINE# h_x = concatenate ( [ x , x_e , h_decoder_h ] , axis = 1 ) #NEWLINE# ( h_encoder_h , h_encoder_c ) = lstm ( h_x , h_encoder_h , h_encoder_c , p . W1 , p . W11 , p . b1 ) #NEWLINE# mu_encoder = ( T . dot ( h_encoder_h , p . W2 ) + p . b2 ) #NEWLINE# log_sigma_encoder = ( 0.5 * ( T . dot ( h_encoder_h , p . W3 ) + p . b3 ) ) #NEWLINE# log_qpz += ( ( - 0.5 ) * T . sum ( ( ( ( 1 + ( 2 * log_sigma_encoder ) ) - ( mu_encoder ** 2 ) ) - T . exp ( ( 2 * log_sigma_encoder ) ) ) ) ) #NEWLINE# z = ( mu_encoder + ( eps [ t ] * T . exp ( log_sigma_encoder ) ) ) #NEWLINE# ( h_decoder_h , h_decoder_c ) = lstm ( z , h_decoder_h , h_decoder_c , p . W4 , p . W44 , p . b4 ) #NEWLINE# c += ( T . dot ( h_decoder_h , p . W5 ) + p . b5 ) #NEWLINE# #UNINDENT# pxz = T . nnet . sigmoid ( c ) #NEWLINE# log_pxz = T . nnet . binary_crossentropy ( pxz , x ) . sum ( ) #NEWLINE# cost = ( log_pxz + log_qpz ) #NEWLINE# z = self . Z . reshape ( ( ( - 1 ) , n_t , n_zpt ) , ndim = 3 ) #NEWLINE# s_ex = p . c0 #NEWLINE# s_h_decoder_h = p . b40_h #NEWLINE# s_h_decoder_c = p . b40_c #NEWLINE# if self . sample_steps : #NEWLINE# #INDENT# a_pxz = T . zeros ( ( ( n_t + 1 ) , z . shape [ 0 ] , n_x ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# a_pxz = T . zeros ( ( 1 , z . shape [ 0 ] , n_x ) ) #NEWLINE# #UNINDENT# for t in xrange ( 0 , n_t ) : #NEWLINE# #INDENT# if self . sample_steps : #NEWLINE# #INDENT# a_pxz = T . set_subtensor ( a_pxz [ t , : , : ] , T . nnet . sigmoid ( s_ex ) ) #NEWLINE# #UNINDENT# ( s_h_decoder_h , s_h_decoder_c ) = lstm ( z [ : , t , : ] , s_h_decoder_h , s_h_decoder_c , p . W4 , p . W44 , p . b4 ) #NEWLINE# s_ex += ( T . dot ( s_h_decoder_h , p . W5 ) + p . b5 ) #NEWLINE# #UNINDENT# a_pxz = T . set_subtensor ( a_pxz [ ( - 1 ) , : , : ] , T . nnet . sigmoid ( s_ex ) ) #NEWLINE# self . compile ( log_pxz , log_qpz , cost , a_pxz )