#NEWLINE# def __init__( self , n_x , n_a , n_z , n_y , qa_hid , qz_hid , qy_hid , px_hid , pa_hid , nonlinearity = rectify , px_nonlinearity = None , x_dist = 'bernoulli' , batchnorm = False , seed = 1234 ) : #NEWLINE# #INDENT# "\n        Initialize an skip deep generative model consisting of\n        discriminative classifier q(y|a,x),\n        generative model P p(a|z,y) and p(x|a,z,y),\n        inference model Q q(a|x) and q(z|a,x,y).\n        Weights are initialized using the Bengio and Glorot (2010) initialization scheme.\n        :param n_x: Number of inputs.\n        :param n_a: Number of auxiliary.\n        :param n_z: Number of latent.\n        :param n_y: Number of classes.\n        :param qa_hid: List of number of deterministic hidden q(a|x).\n        :param qz_hid: List of number of deterministic hidden q(z|a,x,y).\n        :param qy_hid: List of number of deterministic hidden q(y|a,x).\n        :param px_hid: List of number of deterministic hidden p(a|z,y) & p(x|z,y).\n        :param nonlinearity: The transfer function used in the deterministic layers.\n        :param x_dist: The x distribution, 'bernoulli', 'multinomial', or 'gaussian'.\n        :param batchnorm: Boolean value for batch normalization.\n        :param seed: The random seed.\n        " #NEWLINE# super ( SDGMSSL , self ) . __init__ ( n_x , ( qz_hid + px_hid ) , ( n_a + n_z ) , nonlinearity ) #NEWLINE# self . x_dist = x_dist #NEWLINE# self . n_y = n_y #NEWLINE# self . n_x = n_x #NEWLINE# self . n_a = n_a #NEWLINE# self . n_z = n_z #NEWLINE# self . batchnorm = batchnorm #NEWLINE# self . _srng = RandomStreams ( seed ) #NEWLINE# init_w = 0.001 #NEWLINE# hid_w = '' #NEWLINE# if ( ( nonlinearity == rectify ) or ( nonlinearity == softplus ) ) : #NEWLINE# #INDENT# hid_w = 'relu' #NEWLINE# #UNINDENT# self . sym_beta = T . scalar ( 'beta' ) #NEWLINE# self . sym_x_l = T . matrix ( 'x' ) #NEWLINE# self . sym_t_l = T . matrix ( 't' ) #NEWLINE# self . sym_x_u = T . matrix ( 'x' ) #NEWLINE# self . sym_bs_l = T . iscalar ( 'bs_l' ) #NEWLINE# self . sym_samples = T . iscalar ( 'samples' ) #NEWLINE# self . sym_z = T . matrix ( 'z' ) #NEWLINE# self . sym_a = T . matrix ( 'a' ) #NEWLINE# def dense_layer( layer_in , n , dist_w = init . GlorotNormal , dist_b = init . Normal ) : #NEWLINE# #INDENT# dense = DenseLayer ( layer_in , n , dist_w ( hid_w ) , dist_b ( init_w ) , None ) #NEWLINE# if batchnorm : #NEWLINE# #INDENT# dense = BatchNormLayer ( dense ) #NEWLINE# #UNINDENT# return NonlinearityLayer ( dense , self . transf ) #NEWLINE# #UNINDENT# def stochastic_layer( layer_in , n , samples , nonlin = None ) : #NEWLINE# #INDENT# mu = DenseLayer ( layer_in , n , init . Normal ( init_w ) , init . Normal ( init_w ) , nonlin ) #NEWLINE# logvar = DenseLayer ( layer_in , n , init . Normal ( init_w ) , init . Normal ( init_w ) , nonlin ) #NEWLINE# return ( SampleLayer ( mu , logvar , eq_samples = samples , iw_samples = 1 ) , mu , logvar ) #NEWLINE# #UNINDENT# l_x_in = InputLayer ( ( None , n_x ) ) #NEWLINE# l_y_in = InputLayer ( ( None , n_y ) ) #NEWLINE# l_qa_x = l_x_in #NEWLINE# for hid in qa_hid : #NEWLINE# #INDENT# l_qa_x = dense_layer ( l_qa_x , hid ) #NEWLINE# #UNINDENT# ( l_qa_x , l_qa_x_mu , l_qa_x_logvar ) = stochastic_layer ( l_qa_x , n_a , self . sym_samples ) #NEWLINE# l_qa_to_qy = DenseLayer ( l_qa_x , qy_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_qa_to_qy = ReshapeLayer ( l_qa_to_qy , ( ( - 1 ) , self . sym_samples , 1 , qy_hid [ 0 ] ) ) #NEWLINE# l_x_to_qy = DenseLayer ( l_x_in , qy_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_x_to_qy = DimshuffleLayer ( l_x_to_qy , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# l_qy_xa = ReshapeLayer ( ElemwiseSumLayer ( [ l_qa_to_qy , l_x_to_qy ] ) , ( ( - 1 ) , qy_hid [ 0 ] ) ) #NEWLINE# if batchnorm : #NEWLINE# #INDENT# l_qy_xa = BatchNormLayer ( l_qy_xa ) #NEWLINE# #UNINDENT# l_qy_xa = NonlinearityLayer ( l_qy_xa , self . transf ) #NEWLINE# if ( len ( qy_hid ) > 1 ) : #NEWLINE# #INDENT# for hid in qy_hid [ 1 : ] : #NEWLINE# #INDENT# l_qy_xa = dense_layer ( l_qy_xa , hid ) #NEWLINE# #UNINDENT# #UNINDENT# l_qy_xa = DenseLayer ( l_qy_xa , n_y , init . GlorotNormal ( ) , init . Normal ( init_w ) , softmax ) #NEWLINE# l_qa_to_qz = DenseLayer ( l_qa_x , qz_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_qa_to_qz = ReshapeLayer ( l_qa_to_qz , ( ( - 1 ) , self . sym_samples , 1 , qz_hid [ 0 ] ) ) #NEWLINE# l_x_to_qz = DenseLayer ( l_x_in , qz_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_x_to_qz = DimshuffleLayer ( l_x_to_qz , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# l_y_to_qz = DenseLayer ( l_y_in , qz_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_y_to_qz = DimshuffleLayer ( l_y_to_qz , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# l_qz_axy = ReshapeLayer ( ElemwiseSumLayer ( [ l_qa_to_qz , l_x_to_qz , l_y_to_qz ] ) , ( ( - 1 ) , qz_hid [ 0 ] ) ) #NEWLINE# if batchnorm : #NEWLINE# #INDENT# l_qz_axy = BatchNormLayer ( l_qz_axy ) #NEWLINE# #UNINDENT# l_qz_axy = NonlinearityLayer ( l_qz_axy , self . transf ) #NEWLINE# if ( len ( qz_hid ) > 1 ) : #NEWLINE# #INDENT# for hid in qz_hid [ 1 : ] : #NEWLINE# #INDENT# l_qz_axy = dense_layer ( l_qz_axy , hid ) #NEWLINE# #UNINDENT# #UNINDENT# ( l_qz_axy , l_qz_axy_mu , l_qz_axy_logvar ) = stochastic_layer ( l_qz_axy , n_z , 1 ) #NEWLINE# l_y_to_pa = DenseLayer ( l_y_in , pa_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_y_to_pa = DimshuffleLayer ( l_y_to_pa , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# l_qz_to_pa = DenseLayer ( l_qz_axy , pa_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_qz_to_pa = ReshapeLayer ( l_qz_to_pa , ( ( - 1 ) , self . sym_samples , 1 , pa_hid [ 0 ] ) ) #NEWLINE# l_pa_zy = ReshapeLayer ( ElemwiseSumLayer ( [ l_qz_to_pa , l_y_to_pa ] ) , [ ( - 1 ) , pa_hid [ 0 ] ] ) #NEWLINE# if batchnorm : #NEWLINE# #INDENT# l_pa_zy = BatchNormLayer ( l_pa_zy ) #NEWLINE# #UNINDENT# l_pa_zy = NonlinearityLayer ( l_pa_zy , self . transf ) #NEWLINE# if ( len ( pa_hid ) > 1 ) : #NEWLINE# #INDENT# for hid in pa_hid [ 1 : ] : #NEWLINE# #INDENT# l_pa_zy = dense_layer ( l_pa_zy , hid ) #NEWLINE# #UNINDENT# #UNINDENT# ( l_pa_zy , l_pa_zy_mu , l_pa_zy_logvar ) = stochastic_layer ( l_pa_zy , n_a , 1 ) #NEWLINE# l_qa_to_px = DenseLayer ( l_qa_x , px_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_qa_to_px = ReshapeLayer ( l_qa_to_px , ( ( - 1 ) , self . sym_samples , 1 , px_hid [ 0 ] ) ) #NEWLINE# l_y_to_px = DenseLayer ( l_y_in , px_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_y_to_px = DimshuffleLayer ( l_y_to_px , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# l_qz_to_px = DenseLayer ( l_qz_axy , px_hid [ 0 ] , init . GlorotNormal ( hid_w ) , init . Normal ( init_w ) , None ) #NEWLINE# l_qz_to_px = ReshapeLayer ( l_qz_to_px , ( ( - 1 ) , self . sym_samples , 1 , px_hid [ 0 ] ) ) #NEWLINE# l_px_azy = ReshapeLayer ( ElemwiseSumLayer ( [ l_qa_to_px , l_qz_to_px , l_y_to_px ] ) , [ ( - 1 ) , px_hid [ 0 ] ] ) #NEWLINE# if batchnorm : #NEWLINE# #INDENT# l_px_azy = BatchNormLayer ( l_px_azy ) #NEWLINE# #UNINDENT# l_px_azy = NonlinearityLayer ( l_px_azy , self . transf ) #NEWLINE# if ( len ( px_hid ) > 1 ) : #NEWLINE# #INDENT# for hid in px_hid [ 1 : ] : #NEWLINE# #INDENT# l_px_azy = dense_layer ( l_px_azy , hid ) #NEWLINE# #UNINDENT# #UNINDENT# if ( x_dist == 'bernoulli' ) : #NEWLINE# #INDENT# l_px_azy = DenseLayer ( l_px_azy , n_x , init . GlorotNormal ( ) , init . Normal ( init_w ) , sigmoid ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( x_dist == 'multinomial' ) : #NEWLINE# #INDENT# l_px_azy = DenseLayer ( l_px_azy , n_x , init . GlorotNormal ( ) , init . Normal ( init_w ) , softmax ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( x_dist == 'gaussian' ) : #NEWLINE# #INDENT# ( l_px_azy , l_px_zy_mu , l_px_zy_logvar ) = stochastic_layer ( l_px_azy , n_x , 1 , px_nonlinearity ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# self . l_x_in = l_x_in #NEWLINE# self . l_y_in = l_y_in #NEWLINE# self . l_a_in = l_qa_x #NEWLINE# self . l_qa = ReshapeLayer ( l_qa_x , ( ( - 1 ) , self . sym_samples , 1 , n_a ) ) #NEWLINE# self . l_qa_mu = DimshuffleLayer ( l_qa_x_mu , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# self . l_qa_logvar = DimshuffleLayer ( l_qa_x_logvar , ( 0 , 'x' , 'x' , 1 ) ) #NEWLINE# self . l_qz = ReshapeLayer ( l_qz_axy , ( ( - 1 ) , self . sym_samples , 1 , n_z ) ) #NEWLINE# self . l_qz_mu = ReshapeLayer ( l_qz_axy_mu , ( ( - 1 ) , self . sym_samples , 1 , n_z ) ) #NEWLINE# self . l_qz_logvar = ReshapeLayer ( l_qz_axy_logvar , ( ( - 1 ) , self . sym_samples , 1 , n_z ) ) #NEWLINE# self . l_qy = ReshapeLayer ( l_qy_xa , ( ( - 1 ) , self . sym_samples , 1 , n_y ) ) #NEWLINE# self . l_pa = ReshapeLayer ( l_pa_zy , ( ( - 1 ) , self . sym_samples , 1 , n_a ) ) #NEWLINE# self . l_pa_mu = ReshapeLayer ( l_pa_zy_mu , ( ( - 1 ) , self . sym_samples , 1 , n_a ) ) #NEWLINE# self . l_pa_logvar = ReshapeLayer ( l_pa_zy_logvar , ( ( - 1 ) , self . sym_samples , 1 , n_a ) ) #NEWLINE# self . l_px = ReshapeLayer ( l_px_azy , ( ( - 1 ) , self . sym_samples , 1 , n_x ) ) #NEWLINE# self . l_px_mu = ( ReshapeLayer ( l_px_zy_mu , ( ( - 1 ) , l_qz_axy . sym_samples , 1 , n_x ) ) if ( x_dist == 'gaussian' ) else None ) #NEWLINE# self . l_px_logvar = ( ReshapeLayer ( l_px_zy_logvar , ( ( - 1 ) , self . sym_samples , 1 , n_x ) ) if ( x_dist == 'gaussian' ) else None ) #NEWLINE# inputs = [ self . sym_x_l , self . sym_samples ] #NEWLINE# outputs = get_output ( self . l_qy , self . sym_x_l , deterministic = True ) . mean ( axis = ( 1 , 2 ) ) #NEWLINE# self . f_qy = theano . function ( inputs , outputs ) #NEWLINE# inputs = [ self . sym_x_l , self . sym_samples ] #NEWLINE# outputs = get_output ( self . l_qa , self . sym_x_l , deterministic = True ) . mean ( axis = ( 1 , 2 ) ) #NEWLINE# self . f_qa = theano . function ( inputs , outputs ) #NEWLINE# inputs = { #NEWLINE# #INDENT# l_qz_axy : self . sym_z , #NEWLINE# l_y_in : self . sym_t_l , #NEWLINE# #UNINDENT# } #NEWLINE# outputs = get_output ( self . l_pa , inputs , deterministic = True ) #NEWLINE# self . f_pa = theano . function ( [ self . sym_z , self . sym_t_l , self . sym_samples ] , outputs ) #NEWLINE# inputs = { #NEWLINE# #INDENT# l_qa_x : self . sym_a , #NEWLINE# l_qz_axy : self . sym_z , #NEWLINE# l_y_in : self . sym_t_l , #NEWLINE# #UNINDENT# } #NEWLINE# outputs = get_output ( self . l_px , inputs , deterministic = True ) #NEWLINE# self . f_px = theano . function ( [ self . sym_a , self . sym_z , self . sym_t_l , self . sym_samples ] , outputs ) #NEWLINE# self . model_params = get_all_params ( [ self . l_qy , self . l_pa , self . l_px ] ) #NEWLINE# self . trainable_model_params = get_all_params ( [ self . l_qy , self . l_pa , self . l_px ] , trainable = True )