#NEWLINE# def build_model( self , train_set_unlabeled , train_set_labeled , test_set , validation_set = None ) : #NEWLINE# #INDENT# '\n        Build the auxiliary deep generative model from the initialized hyperparameters.\n        Define the lower bound term and compile it into a training function.\n        :param train_set_unlabeled: Unlabeled train set containing variables x, t.\n        :param train_set_labeled: Unlabeled train set containing variables x, t.\n        :param test_set: Test set containing variables x, t.\n        :param validation_set: Validation set containing variables x, t.\n        :return: train, test, validation function and dicts of arguments.\n        ' #NEWLINE# super ( SDGMSSL , self ) . build_model ( train_set_unlabeled , test_set , validation_set ) #NEWLINE# sh_train_x_l = theano . shared ( np . asarray ( train_set_labeled [ 0 ] , dtype = theano . config . floatX ) , borrow = True ) #NEWLINE# sh_train_t_l = theano . shared ( np . asarray ( train_set_labeled [ 1 ] , dtype = theano . config . floatX ) , borrow = True ) #NEWLINE# n = self . sh_train_x . shape [ 0 ] . astype ( theano . config . floatX ) #NEWLINE# n_l = sh_train_x_l . shape [ 0 ] . astype ( theano . config . floatX ) #NEWLINE# l_log_qa = GaussianLogDensityLayer ( self . l_qa , self . l_qa_mu , self . l_qa_logvar ) #NEWLINE# l_log_qz = GaussianLogDensityLayer ( self . l_qz , self . l_qz_mu , self . l_qz_logvar ) #NEWLINE# l_log_qy = MultinomialLogDensityLayer ( self . l_qy , self . l_y_in , eps = 1e-08 ) #NEWLINE# l_log_pz = StandardNormalLogDensityLayer ( self . l_qz ) #NEWLINE# l_log_pa = GaussianLogDensityLayer ( self . l_qa , self . l_pa_mu , self . l_pa_logvar ) #NEWLINE# if ( self . x_dist == 'bernoulli' ) : #NEWLINE# #INDENT# l_log_px = BernoulliLogDensityLayer ( self . l_px , self . l_x_in ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( self . x_dist == 'multinomial' ) : #NEWLINE# #INDENT# l_log_px = MultinomialLogDensityLayer ( self . l_px , self . l_x_in ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( self . x_dist == 'gaussian' ) : #NEWLINE# #INDENT# l_log_px = GaussianLogDensityLayer ( self . l_x_in , self . l_px_mu , self . l_px_logvar ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# def lower_bound( log_pa , log_qa , log_pz , log_qz , log_py , log_px ) : #NEWLINE# #INDENT# lb = ( ( ( ( ( log_px + log_py ) + log_pz ) + log_pa ) - log_qa ) - log_qz ) #NEWLINE# return lb #NEWLINE# #UNINDENT# out_layers = [ l_log_pa , l_log_pz , l_log_qa , l_log_qz , l_log_px , l_log_qy ] #NEWLINE# inputs = { #NEWLINE# #INDENT# self . l_x_in : self . sym_x_l , #NEWLINE# self . l_y_in : self . sym_t_l , #NEWLINE# #UNINDENT# } #NEWLINE# out = get_output ( out_layers , inputs , batch_norm_update_averages = False , batch_norm_use_averages = False ) #NEWLINE# ( log_pa_l , log_pz_l , log_qa_x_l , log_qz_axy_l , log_px_zy_l , log_qy_ax_l ) = out #NEWLINE# py_l = softmax ( T . zeros ( ( self . sym_x_l . shape [ 0 ] , self . n_y ) ) ) #NEWLINE# log_py_l = ( - categorical_crossentropy ( py_l , self . sym_t_l ) . reshape ( ( ( - 1 ) , 1 ) ) . dimshuffle ( ( 0 , 'x' , 'x' , 1 ) ) ) #NEWLINE# lb_l = lower_bound ( log_pa_l , log_qa_x_l , log_pz_l , log_qz_axy_l , log_py_l , log_px_zy_l ) #NEWLINE# lb_l = lb_l . mean ( axis = ( 1 , 2 ) ) #NEWLINE# log_qy_ax_l *= ( self . sym_beta * ( n / n_l ) ) #NEWLINE# lb_l -= log_qy_ax_l . mean ( axis = ( 1 , 2 ) ) #NEWLINE# bs_u = self . sym_x_u . shape [ 0 ] #NEWLINE# t_eye = T . eye ( self . n_y , k = 0 ) #NEWLINE# t_u = t_eye . reshape ( ( self . n_y , 1 , self . n_y ) ) . repeat ( bs_u , axis = 1 ) . reshape ( ( ( - 1 ) , self . n_y ) ) #NEWLINE# x_u = self . sym_x_u . reshape ( ( 1 , bs_u , self . n_x ) ) . repeat ( self . n_y , axis = 0 ) . reshape ( ( ( - 1 ) , self . n_x ) ) #NEWLINE# a_x_u = get_output ( self . l_qa , self . sym_x_u , batch_norm_update_averages = True , batch_norm_use_averages = False ) #NEWLINE# a_x_u_rep = a_x_u . reshape ( ( 1 , ( bs_u * self . sym_samples ) , self . n_a ) ) . repeat ( self . n_y , axis = 0 ) . reshape ( ( ( - 1 ) , self . n_a ) ) #NEWLINE# out_layers = [ l_log_pa , l_log_pz , l_log_qa , l_log_qz , l_log_px ] #NEWLINE# inputs = { #NEWLINE# #INDENT# self . l_x_in : x_u , #NEWLINE# self . l_y_in : t_u , #NEWLINE# self . l_a_in : a_x_u_rep , #NEWLINE# #UNINDENT# } #NEWLINE# out = get_output ( out_layers , inputs , batch_norm_update_averages = False , batch_norm_use_averages = False ) #NEWLINE# ( log_pa_u , log_pz_u , log_qa_x_u , log_qz_axy_u , log_px_zy_u ) = out #NEWLINE# py_u = softmax ( T . zeros ( ( ( bs_u * self . n_y ) , self . n_y ) ) ) #NEWLINE# log_py_u = ( - categorical_crossentropy ( py_u , t_u ) . reshape ( ( ( - 1 ) , 1 ) ) . dimshuffle ( ( 0 , 'x' , 'x' , 1 ) ) ) #NEWLINE# lb_u = lower_bound ( log_pa_u , log_qa_x_u , log_pz_u , log_qz_axy_u , log_py_u , log_px_zy_u ) #NEWLINE# lb_u = lb_u . reshape ( ( self . n_y , 1 , 1 , bs_u ) ) . transpose ( 3 , 1 , 2 , 0 ) . mean ( axis = ( 1 , 2 ) ) #NEWLINE# inputs = { #NEWLINE# #INDENT# self . l_x_in : self . sym_x_u , #NEWLINE# self . l_a_in : a_x_u . reshape ( ( ( - 1 ) , self . n_a ) ) , #NEWLINE# #UNINDENT# } #NEWLINE# y_u = get_output ( self . l_qy , inputs , batch_norm_update_averages = True , batch_norm_use_averages = False ) . mean ( axis = ( 1 , 2 ) ) #NEWLINE# y_u += 1e-08 #NEWLINE# y_u /= T . sum ( y_u , axis = 1 , keepdims = True ) #NEWLINE# lb_u = ( y_u * ( lb_u - T . log ( y_u ) ) ) . sum ( axis = 1 ) #NEWLINE# if self . batchnorm : #NEWLINE# #INDENT# inputs = { #NEWLINE# #INDENT# self . l_x_in : self . sym_x_u , #NEWLINE# self . l_y_in : y_u , #NEWLINE# self . l_a_in : a_x_u , #NEWLINE# #UNINDENT# } #NEWLINE# get_output ( out_layers , inputs , weighting = None , batch_norm_update_averages = True , batch_norm_use_averages = False ) #NEWLINE# #UNINDENT# weight_priors = 0.0 #NEWLINE# for p in self . trainable_model_params : #NEWLINE# #INDENT# if ( 'W' not in str ( p ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# weight_priors += log_normal ( p , 0 , 1 ) . sum ( ) #NEWLINE# #UNINDENT# elbo = ( ( ( ( lb_l . mean ( ) + lb_u . mean ( ) ) * n ) + weight_priors ) / ( - n ) ) #NEWLINE# lb_labeled = ( - lb_l . mean ( ) ) #NEWLINE# lb_unlabeled = ( - lb_u . mean ( ) ) #NEWLINE# grads_collect = T . grad ( elbo , self . trainable_model_params ) #NEWLINE# params_collect = self . trainable_model_params #NEWLINE# sym_beta1 = T . scalar ( 'beta1' ) #NEWLINE# sym_beta2 = T . scalar ( 'beta2' ) #NEWLINE# ( clip_grad , max_norm ) = ( 1 , 5 ) #NEWLINE# mgrads = total_norm_constraint ( grads_collect , max_norm = max_norm ) #NEWLINE# mgrads = [ T . clip ( g , ( - clip_grad ) , clip_grad ) for g in mgrads ] #NEWLINE# updates = adam ( mgrads , params_collect , self . sym_lr , sym_beta1 , sym_beta2 ) #NEWLINE# indices = self . _srng . choice ( size = [ self . sym_bs_l ] , a = sh_train_x_l . shape [ 0 ] , replace = False ) #NEWLINE# x_batch_l = sh_train_x_l [ indices ] #NEWLINE# t_batch_l = sh_train_t_l [ indices ] #NEWLINE# x_batch_u = self . sh_train_x [ self . batch_slice ] #NEWLINE# if ( self . x_dist == 'bernoulli' ) : #NEWLINE# #INDENT# x_batch_u = self . _srng . binomial ( size = x_batch_u . shape , n = 1 , p = x_batch_u , dtype = theano . config . floatX ) #NEWLINE# x_batch_l = self . _srng . binomial ( size = train_set_unlabeled . shape , n = 1 , p = x_batch_l , dtype = theano . config . floatX ) #NEWLINE# #UNINDENT# givens = { #NEWLINE# #INDENT# self . sym_x_l : x_batch_l , #NEWLINE# self . sym_x_u : x_batch_u , #NEWLINE# self . sym_t_l : t_batch_l , #NEWLINE# #UNINDENT# } #NEWLINE# inputs = [ self . sym_index , self . sym_batchsize , self . sym_bs_l , self . sym_beta , self . sym_lr , sym_beta1 , sym_beta2 , self . sym_samples ] #NEWLINE# outputs = [ elbo , lb_labeled , lb_unlabeled ] #NEWLINE# f_train = theano . function ( inputs = inputs , outputs = outputs , givens = givens , updates = updates ) #NEWLINE# self . train_args [ 'inputs' ] [ 'batchsize_unlabeled' ] = 100 #NEWLINE# self . train_args [ 'inputs' ] [ 'batchsize_labeled' ] = 100 #NEWLINE# self . train_args [ 'inputs' ] [ 'beta' ] = 0.1 #NEWLINE# self . train_args [ 'inputs' ] [ 'learningrate' ] = 0.0003 #NEWLINE# self . train_args [ 'inputs' ] [ 'beta1' ] = 0.9 #NEWLINE# self . train_args [ 'inputs' ] [ 'beta2' ] = 0.999 #NEWLINE# self . train_args [ 'inputs' ] [ 'samples' ] = 1 #NEWLINE# self . train_args [ 'outputs' ] [ 'lb' ] = '%0.4f' #NEWLINE# self . train_args [ 'outputs' ] [ 'lb-labeled' ] = '%0.4f' #NEWLINE# self . train_args [ 'outputs' ] [ 'lb-unlabeled' ] = '%0.4f' #NEWLINE# y = get_output ( self . l_qy , self . sym_x_l , deterministic = True ) . mean ( axis = ( 1 , 2 ) ) #NEWLINE# class_err = ( ( 1.0 - categorical_accuracy ( y , self . sym_t_l ) . mean ( ) ) * 100 ) #NEWLINE# givens = { #NEWLINE# #INDENT# self . sym_x_l : self . sh_test_x , #NEWLINE# self . sym_t_l : self . sh_test_t , #NEWLINE# #UNINDENT# } #NEWLINE# f_test = theano . function ( inputs = [ self . sym_samples ] , outputs = [ class_err ] , givens = givens ) #NEWLINE# self . test_args [ 'inputs' ] [ 'samples' ] = 1 #NEWLINE# self . test_args [ 'outputs' ] [ 'test' ] = '%0.2f%%' #NEWLINE# f_validate = None #NEWLINE# if ( validation_set is not None ) : #NEWLINE# #INDENT# givens = { #NEWLINE# #INDENT# self . sym_x_l : self . sh_valid_x , #NEWLINE# self . sym_t_l : self . sh_valid_t , #NEWLINE# #UNINDENT# } #NEWLINE# f_validate = theano . function ( inputs = [ self . sym_samples ] , outputs = [ class_err ] , givens = givens ) #NEWLINE# #UNINDENT# self . validate_args [ 'inputs' ] [ 'samples' ] = 1 #NEWLINE# self . validate_args [ 'outputs' ] [ 'validation' ] = '%0.2f%%' #NEWLINE# return ( f_train , f_test , f_validate , self . train_args , self . test_args , self . validate_args )