#NEWLINE# def __init__( self , data , hp ) : #NEWLINE# #INDENT# super ( LM_draw , self ) . __init__ ( self . __class__ . __name__ , data , hp ) #NEWLINE# self . n_h = 1024 #NEWLINE# self . n_zpt = 256 #NEWLINE# self . dropout = 0.0 #NEWLINE# self . params = Parameters ( ) #NEWLINE# self . hiddenstates = Parameters ( ) #NEWLINE# n_tokens = self . data [ 'n_tokens' ] #NEWLINE# gates = 4 #NEWLINE# with self . hiddenstates : #NEWLINE# #INDENT# b1_h = shared_zeros ( ( self . hp . batch_size , self . n_h ) ) #NEWLINE# b1_c = shared_zeros ( ( self . hp . batch_size , self . n_h ) ) #NEWLINE# b2_h = shared_zeros ( ( self . hp . batch_size , self . n_h ) ) #NEWLINE# b2_c = shared_zeros ( ( self . hp . batch_size , self . n_h ) ) #NEWLINE# #UNINDENT# if ( hp . load_model and os . path . isfile ( self . filename ) ) : #NEWLINE# #INDENT# self . params . load ( self . filename ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# with self . params : #NEWLINE# #INDENT# W_emb = shared_normal ( ( n_tokens , self . n_h ) , scale = hp . init_scale ) #NEWLINE# W1 = shared_normal ( ( ( self . n_h * 2 ) , ( self . n_h * gates ) ) , scale = ( hp . init_scale * 1.5 ) ) #NEWLINE# V1 = shared_normal ( ( self . n_h , ( self . n_h * gates ) ) , scale = ( hp . init_scale * 1.5 ) ) #NEWLINE# b1 = shared_zeros ( ( ( self . n_h * gates ) , ) ) #NEWLINE# Wmu = shared_normal ( ( self . n_h , self . n_zpt ) , scale = hp . init_scale ) #NEWLINE# Wsi = shared_normal ( ( self . n_h , self . n_zpt ) , scale = hp . init_scale ) #NEWLINE# bmu = shared_zeros ( ( self . n_zpt , ) ) #NEWLINE# bsi = shared_zeros ( ( self . n_zpt , ) ) #NEWLINE# W2 = shared_normal ( ( self . n_zpt , ( self . n_h * gates ) ) , scale = ( hp . init_scale * 1.5 ) ) #NEWLINE# V2 = shared_normal ( ( self . n_h , ( self . n_h * gates ) ) , scale = ( hp . init_scale * 1.5 ) ) #NEWLINE# b2 = shared_zeros ( ( ( self . n_h * gates ) , ) ) #NEWLINE# #UNINDENT# #UNINDENT# def lstm( X , h , c , W , U , b ) : #NEWLINE# #INDENT# g_on = ( ( T . dot ( X , W ) + T . dot ( h , U ) ) + b ) #NEWLINE# i_on = T . nnet . sigmoid ( g_on [ : , : self . n_h ] ) #NEWLINE# f_on = T . nnet . sigmoid ( g_on [ : , self . n_h : ( 2 * self . n_h ) ] ) #NEWLINE# o_on = T . nnet . sigmoid ( g_on [ : , ( 2 * self . n_h ) : ( 3 * self . n_h ) ] ) #NEWLINE# c = ( ( f_on * c ) + ( i_on * T . tanh ( g_on [ : , ( 3 * self . n_h ) : ] ) ) ) #NEWLINE# h = ( o_on * T . tanh ( c ) ) #NEWLINE# return ( h , c ) #NEWLINE# #UNINDENT# def model( x , p , p_dropout , noise ) : #NEWLINE# #INDENT# input_size = x . shape [ 1 ] #NEWLINE# h0 = p . W_emb [ x ] #NEWLINE# h0 = dropout ( h0 , p_dropout ) #NEWLINE# ( cost , h1 , c1 , h2 , c2 ) = [ 0.0 , b1_h , b1_c , b2_h , b2_c ] #NEWLINE# eps = srnd . normal ( ( self . hp . seq_size , input_size , self . n_zpt ) , dtype = theano . config . floatX ) #NEWLINE# for t in xrange ( 0 , self . hp . seq_size ) : #NEWLINE# #INDENT# if ( t >= self . hp . warmup_size ) : #NEWLINE# #INDENT# pyx = softmax ( T . dot ( h2 , T . transpose ( p . W_emb ) ) ) #NEWLINE# cost += T . sum ( T . nnet . categorical_crossentropy ( pyx , theano_one_hot ( x [ t ] , n_tokens ) ) ) #NEWLINE# #UNINDENT# h_x = concatenate ( [ h0 [ t ] , h2 ] , axis = 1 ) #NEWLINE# ( h1 , c1 ) = lstm ( h_x , h1 , c1 , p . W1 , p . V1 , p . b1 ) #NEWLINE# h1 = dropout ( h1 , p_dropout ) #NEWLINE# mu_encoder = ( T . dot ( h1 , p . Wmu ) + p . bmu ) #NEWLINE# if noise : #NEWLINE# #INDENT# log_sigma_encoder = ( 0.5 * ( T . dot ( h1 , p . Wsi ) + p . bsi ) ) #NEWLINE# cost += ( ( ( - 0.5 ) * T . sum ( ( ( ( 1 + ( 2 * log_sigma_encoder ) ) - ( mu_encoder ** 2 ) ) - T . exp ( ( 2 * log_sigma_encoder ) ) ) ) ) * 0.01 ) #NEWLINE# z = ( mu_encoder + ( eps [ t ] * T . exp ( log_sigma_encoder ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# z = mu_encoder #NEWLINE# #UNINDENT# ( h2 , c2 ) = lstm ( z , h2 , c2 , p . W2 , p . V2 , p . b2 ) #NEWLINE# h2 = dropout ( h2 , p_dropout ) #NEWLINE# #UNINDENT# h_updates = [ ( b1_h , h1 ) , ( b1_c , c1 ) , ( b2_h , h2 ) , ( b2_c , c2 ) ] #NEWLINE# return ( cost , h_updates ) #NEWLINE# #UNINDENT# ( cost , h_updates ) = model ( self . X , self . params , self . dropout , True ) #NEWLINE# ( te_cost , te_h_updates ) = model ( self . X , self . params , 0.0 , False ) #NEWLINE# def generate( seed_idx , p ) : #NEWLINE# #INDENT# spx = T . zeros ( ( self . hp . seq_size , n_tokens ) ) #NEWLINE# ( h1 , c1 , h2 , c2 ) = [ T . zeros ( self . n_h ) , T . zeros ( self . n_h ) , T . zeros ( self . n_h ) , T . zeros ( self . n_h ) ] #NEWLINE# spx = T . set_subtensor ( spx [ ( 0 , seed_idx ) ] , 1 ) #NEWLINE# return spx #NEWLINE# #UNINDENT# spx = generate ( self . seed_idx , self . params ) #NEWLINE# self . compile ( cost , te_cost , h_updates , te_h_updates , spx )