jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def evaluate():
    """Evaluated current agent, and records a video with it's performance"""
    envwrap = GymWrapperFactory.make(FLAGS.env,
                                     actrep=FLAGS.action_repeat,
                                     memlen=FLAGS.memory_len,
                                     w=FLAGS.width,
                                     h=FLAGS.height)
    with tf.Session() as sess:
        agent = QlearningAgent(session=sess,
                               action_size=envwrap.action_size,
                               h=FLAGS.height,
                               w=FLAGS.width,
                               channels=FLAGS.memory_len,
                               opt=tf.train.AdamOptimizer(FLAGS.lr))
        sess.run(tf.global_variables_initializer())
        if not os.path.exists(FLAGS.logdir):
            print('ERROR! No', FLAGS.logdir, 'folder found!')
            return
        ckpt = tf.train.latest_checkpoint(FLAGS.logdir)
        if ckpt is not None:
            tf.train.Saver().restore(sess, ckpt)
            agent.update_target()
            print('Session was restored from %s' % ckpt)
        else:
            print('ERROR! No checkpoint found at', FLAGS.logdir)
            return
        envwrap.env.monitor.start(os.path.join(FLAGS.evaldir, FLAGS.env))
        total_reward = 0
        for _ in range(FLAGS.eval_iter):
            s = envwrap.reset()
            terminal = False
            while not terminal:
                reward_per_action = agent.predict_rewards(s)
                s, r, terminal, info = envwrap.step(np.argmax(reward_per_action), test=True)
                total_reward += r
                envwrap.render()
        envwrap.env.monitor.close()
        print('Evaluation finished.')
        print('Average reward per episode: %0.4f' % (total_reward / FLAGS.eval_iter))
//  [python]
def initialize_new(self, session=None):
    """Saves the session, initializes all variables to random values.

    Args:
      session: A tensorflow session.
    """
    with self.graph.as_default():
      if session is None:
        self.session = tf.Session(graph=self.graph)
      else:
        self.session = session
      self.session.run(tf.initialize_all_variables())
//  [python]
def initialize_new(self, session=None):
    """Saves the session, initializes all variables to random values.

    Args:
      session: A tensorflow session.
    """
    with self.graph.as_default():
      if session is None:
        self.session = tf.Session(graph=self.graph)
      else:
        self.session = session
      self.session.run(tf.global_variables_initializer())
//  [python]
def __init__(self,sess,state_dim,action_dim):
        self.time_step = 0
        self.sess = sess
        # create q network
        self.state_input,        self.action_input,        self.q_value_output,        self.net,        self.is_training = self.create_q_network(state_dim,action_dim)

        # create target q network (the same structure with q network)
        self.target_state_input,        self.target_action_input,        self.target_q_value_output,        self.target_update,        self.target_is_training = self.create_target_q_network(state_dim,action_dim,self.net)

        self.create_training_method()

        # initialization 
        self.sess.run(tf.initialize_all_variables())

        self.update_target()
//  [python]
def __init__(self,sess,state_dim,action_dim):
        self.time_step = 0
        self.sess = sess
        # create q network
        self.state_input,        self.action_input,        self.q_value_output,        self.net,        self.is_training = self.create_q_network(state_dim,action_dim)

        # create target q network (the same structure with q network)
        self.target_state_input,        self.target_action_input,        self.target_q_value_output,        self.target_update,        self.target_is_training = self.create_target_q_network(state_dim,action_dim,self.net)

        self.create_training_method()

        # initialization 
        self.sess.run(tf.global_variables_initializer())

        self.update_target()
//  [python]
state = tf.Variable(0, name='counter')
#print(state.name)
one = tf.constant(1)

new_value = tf.add(state, one)
update = tf.assign(state, new_value)

# tf.initialize_all_variables() no long valid from
# 2017-03-02 if using tensorflow >= 0.12
if int((tf.__version__).split('.')[1]) < 12 and int((tf.__version__).split('.')[0]) < 1:
    init = tf.initialize_all_variables()
else:
    init = tf.global_variables_initializer()
//  [python]
state = tf.Variable(0, name='counter')
#print(state.name)
one = tf.constant(1)

new_value = tf.add(state, one)
update = tf.assign(state, new_value)

init = tf.global_variables_initializer()
//  [python]
e4 = tf.as_string(eee)
global_step = tf.train.get_or_create_global_step()
global_step_inc = tf.assign_add(global_step, 1)
hooks = [tf.train.StopAtStepHook(last_step=2)]
with tf.Session() as mon_sess:
    mon_sess.run(tf.initialize_all_variables())
    for i in range(2):
        print(mon_sess.run([d, e4, global_step_inc], feed_dict={a: [1.0, 2.0, 3.0], b: [1.0, 2.0, 3.0],
                                                                e: ["1.0", "2.0", "3.0"]}))

    signatures = {
        tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: {
            'inputs': {'a': a, "b": b, "e": e},
            'outputs': {"d": d, "e4": e4},
            'method_name': tf.saved_model.signature_constants.PREDICT_METHOD_NAME
        }
    }

    export_dir = os.path.dirname(os.path.abspath(__file__)) + "../../../../target/test-classes/export2"
    try:
        tf.gfile.DeleteRecursively(export_dir)
    except tf.errors.OpError:
        pass
    builder = tf.saved_model.builder.SavedModelBuilder(export_dir)

    signature_def_map = {}
    for key, sig in signatures.items():
        signature_def_map[key] = tf.saved_model.signature_def_utils.build_signature_def(
            inputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                    sig['inputs'].items()},
            outputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                     sig['outputs'].items()},
            method_name=sig['method_name'] if 'method_name' in sig else key)

        builder.add_meta_graph_and_variables(
            mon_sess,
            tf.saved_model.tag_constants.SERVING.split(','),
            signature_def_map=signature_def_map,
            clear_devices=True)
        builder.save()
//  [python]
e4 = tf.as_string(eee)
global_step = tf.train.get_or_create_global_step()
global_step_inc = tf.assign_add(global_step, 1)
hooks = [tf.train.StopAtStepHook(last_step=2)]
with tf.Session() as mon_sess:
    mon_sess.run(tf.global_variables_initializer())
    for i in range(2):
        print(mon_sess.run([d, e4, global_step_inc], feed_dict={a: [1.0, 2.0, 3.0], b: [1.0, 2.0, 3.0],
                                                                e: ["1.0", "2.0", "3.0"]}))

    signatures = {
        tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: {
            'inputs': {'a': a, "b": b, "e": e},
            'outputs': {"d": d, "e4": e4},
            'method_name': tf.saved_model.signature_constants.PREDICT_METHOD_NAME
        }
    }

    export_dir = os.path.dirname(os.path.abspath(__file__)) + "../../../../target/test-classes/export2"
    try:
        tf.gfile.DeleteRecursively(export_dir)
    except tf.errors.OpError:
        pass
    builder = tf.saved_model.builder.SavedModelBuilder(export_dir)

    signature_def_map = {}
    for key, sig in signatures.items():
        signature_def_map[key] = tf.saved_model.signature_def_utils.build_signature_def(
            inputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                    sig['inputs'].items()},
            outputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                     sig['outputs'].items()},
            method_name=sig['method_name'] if 'method_name' in sig else key)

        builder.add_meta_graph_and_variables(
            mon_sess,
            tf.saved_model.tag_constants.SERVING.split(','),
            signature_def_map=signature_def_map,
            clear_devices=True)
        builder.save()
//  [python]
def main(unused_argv=None):
  obj = create_hub_module_object()
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    tf.train.Saver(obj.variables).restore(sess, FLAGS.checkpoint)
    tf.saved_model.save(obj, FLAGS.export_path, signatures=obj.__call__)
  tf.logging.info('Saved hub module in: %s', FLAGS.export_path)
//  [python]
def main(unused_argv=None):
  obj = create_hub_module_object()
  with tf.compat.v1.Session() as sess:
    sess.run(tf.compat.v1.global_variables_initializer())
    tf.train.Saver(obj.variables).restore(sess, FLAGS.checkpoint)
    tf.saved_model.save(obj, FLAGS.export_path, signatures=obj.__call__)
  tf.logging.info('Saved hub module in: %s', FLAGS.export_path)
//  [python]
kspace_und_re = np.reshape(kspace_und_re,[1,dim_kspaceUnd_X,dim_kspaceUnd_Y,dim_kspaceUnd_Z*2])
kspace_recon = kspace_und_re

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 1/3 ; 

for ind_c in range(0,no_channels):
    print('Reconstruting Channel #',ind_c+1)
    
    sess = tf.Session(config=config) 
    if int((tf.__version__).split('.')[1]) < 12 and int((tf.__version__).split('.')[0]) < 1:
        init = tf.initialize_all_variables()
    else:
        init = tf.global_variables_initializer()
    sess.run(init)
    
    # grab w and b
    w1 = np.float32(w1_all[:,:,:,:,ind_c])
    w2 = np.float32(w2_all[:,:,:,:,ind_c])     
    w3 = np.float32(w3_all[:,:,:,:,ind_c])

    if (b1_flag == 1):
        b1 = b1_all[:,:,:,ind_c];
    if (b2_flag == 1):
        b2 = b2_all[:,:,:,ind_c];
    if (b3_flag == 1):
        b3 = b3_all[:,:,:,ind_c];                
        
    res = cnn_3layer(kspace_und_re,w1,b1,w2,b2,w3,b3,acc_rate,sess) 
    target_x_end_kspace = dim_kspaceUnd_X - target_x_start;
    
    for ind_acc in range(0,acc_rate-1):

        target_y_start = np.int32((np.ceil(kernel_y_1/2)-1) + np.int32((np.ceil(kernel_y_2/2)-1)) + np.int32(np.ceil(kernel_last_y/2)-1)) * acc_rate + ind_acc + 1;             
        target_y_end_kspace = dim_kspaceUnd_Y - np.int32((np.floor(kernel_y_1/2)) + (np.floor(kernel_y_2/2)) + np.floor(kernel_last_y/2)) * acc_rate + ind_acc;
        kspace_recon[0,target_x_start:target_x_end_kspace,target_y_start:target_y_end_kspace+1:acc_rate,ind_c] = res[0,:,::acc_rate,ind_acc]

    sess.close()
    tf.reset_default_graph()
//  [python]
kspace_und_re = np.reshape(kspace_und_re,[1,dim_kspaceUnd_X,dim_kspaceUnd_Y,dim_kspaceUnd_Z*2])
kspace_recon = kspace_und_re

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 1/3 ; 

for ind_c in range(0,no_channels):
    print('Reconstruting Channel #',ind_c+1)
    
    sess = tf.Session(config=config) 
    init = tf.global_variables_initializer()
    sess.run(init)
    
    # grab w and b
    w1 = np.float32(w1_all[:,:,:,:,ind_c])
    w2 = np.float32(w2_all[:,:,:,:,ind_c])     
    w3 = np.float32(w3_all[:,:,:,:,ind_c])

    if (b1_flag == 1):
        b1 = b1_all[:,:,:,ind_c];
    if (b2_flag == 1):
        b2 = b2_all[:,:,:,ind_c];
    if (b3_flag == 1):
        b3 = b3_all[:,:,:,ind_c];                
        
    res = cnn_3layer(kspace_und_re,w1,b1,w2,b2,w3,b3,acc_rate,sess) 
    target_x_end_kspace = dim_kspaceUnd_X - target_x_start;
    
    for ind_acc in range(0,acc_rate-1):

        target_y_start = np.int32((np.ceil(kernel_y_1/2)-1) + np.int32((np.ceil(kernel_y_2/2)-1)) + np.int32(np.ceil(kernel_last_y/2)-1)) * acc_rate + ind_acc + 1;             
        target_y_end_kspace = dim_kspaceUnd_Y - np.int32((np.floor(kernel_y_1/2)) + (np.floor(kernel_y_2/2)) + np.floor(kernel_last_y/2)) * acc_rate + ind_acc;
        kspace_recon[0,target_x_start:target_x_end_kspace,target_y_start:target_y_end_kspace+1:acc_rate,ind_c] = res[0,:,::acc_rate,ind_acc]

    sess.close()
    tf.reset_default_graph()
//  [python]
def learning(ACS_input,target_input,accrate_input,sess):
    
    input_ACS = tf.placeholder(tf.float32, [1, ACS_dim_X,ACS_dim_Y,ACS_dim_Z])                                  
    input_Target = tf.placeholder(tf.float32, [1, target_dim_X,target_dim_Y,target_dim_Z])         
    
    Input = tf.reshape(input_ACS, [1, ACS_dim_X, ACS_dim_Y, ACS_dim_Z])         

    [target_dim0,target_dim1,target_dim2,target_dim3] = np.shape(target)

    W_conv1 = weight_variable([kernel_x_1, kernel_y_1, ACS_dim_Z, layer1_channels],'W1') 
    h_conv1 = tf.nn.relu(conv2d_dilate(Input, W_conv1,accrate_input)) 

    W_conv2 = weight_variable([kernel_x_2, kernel_y_2, layer1_channels, layer2_channels],'W2')
    h_conv2 = tf.nn.relu(conv2d_dilate(h_conv1, W_conv2,accrate_input))

    W_conv3 = weight_variable([kernel_last_x, kernel_last_y, layer2_channels, target_dim3],'W3')
    h_conv3 = conv2d_dilate(h_conv2, W_conv3,accrate_input)

    error_norm = tf.norm(input_Target - h_conv3)       
    train_step = tf.train.AdamOptimizer(LearningRate).minimize(error_norm)

    if int((tf.__version__).split('.')[1]) < 12 and int((tf.__version__).split('.')[0]) < 1:
        init = tf.initialize_all_variables()
    else:
        init = tf.global_variables_initializer()
    sess.run(init)

    error_prev = 1 
    for i in range(MaxIteration):
        
        sess.run(train_step, feed_dict={input_ACS: ACS, input_Target: target})
        if i % 100 == 0:                                                                      
            error_now=sess.run(error_norm,feed_dict={input_ACS: ACS, input_Target: target})    
            print('The',i,'th iteration gives an error',error_now)                             
            
            
        
    error = sess.run(error_norm,feed_dict={input_ACS: ACS, input_Target: target})
    return [sess.run(W_conv1),sess.run(W_conv2),sess.run(W_conv3),error]
//  [python]
def learning(ACS_input,target_input,accrate_input,sess):
    
    input_ACS = tf.placeholder(tf.float32, [1, ACS_dim_X,ACS_dim_Y,ACS_dim_Z])                                  
    input_Target = tf.placeholder(tf.float32, [1, target_dim_X,target_dim_Y,target_dim_Z])         
    
    Input = tf.reshape(input_ACS, [1, ACS_dim_X, ACS_dim_Y, ACS_dim_Z])         

    [target_dim0,target_dim1,target_dim2,target_dim3] = np.shape(target)

    W_conv1 = weight_variable([kernel_x_1, kernel_y_1, ACS_dim_Z, layer1_channels],'W1') 
    h_conv1 = tf.nn.relu(conv2d_dilate(Input, W_conv1,accrate_input)) 

    W_conv2 = weight_variable([kernel_x_2, kernel_y_2, layer1_channels, layer2_channels],'W2')
    h_conv2 = tf.nn.relu(conv2d_dilate(h_conv1, W_conv2,accrate_input))

    W_conv3 = weight_variable([kernel_last_x, kernel_last_y, layer2_channels, target_dim3],'W3')
    h_conv3 = conv2d_dilate(h_conv2, W_conv3,accrate_input)

    error_norm = tf.norm(input_Target - h_conv3)       
    train_step = tf.train.AdamOptimizer(LearningRate).minimize(error_norm)

    init = tf.global_variables_initializer()
    sess.run(init)

    error_prev = 1 
    for i in range(MaxIteration):
        
        sess.run(train_step, feed_dict={input_ACS: ACS, input_Target: target})
        if i % 100 == 0:                                                                      
            error_now=sess.run(error_norm,feed_dict={input_ACS: ACS, input_Target: target})    
            print('The',i,'th iteration gives an error',error_now)                             
            
            
        
    error = sess.run(error_norm,feed_dict={input_ACS: ACS, input_Target: target})
    return [sess.run(W_conv1),sess.run(W_conv2),sess.run(W_conv3),error]
//  [python]
def Data_Learning(x_data, y_data):
    W = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
    # init = tf.initialize_all_variables()
    # sess = tf.Session()
    # sess.run(init)
    # sess.run(W)
    # print('sess.run(W)= ', sess.run(W))
    # array([0.05211711], dtype=float32)

    b = tf.Variable(tf.zeros([1]))
    # init = tf.initialize_all_variables()
    # sess = tf.Session()
    # sess.run(init)
    # sess.run(b)
    # array([0.], dtype=float32)

    y = W * x_data + b
    # sess.run(y)
    # print(np.round(sess.run(y),1))

    loss = tf.reduce_mean(tf.square(y - y_data))
    # print(np.round(sess.run(loss),1))
    # optimizer = tf.train.GradientDescentOptimizer(0.0015)
    #
    #optimizer = tf.train.GradientDescentOptimizer(0.0001)
    # 느린 수렴 : (0.0001) --> for step in np.arange(1000):
    #

    optimizer = tf.train.GradientDescentOptimizer(0.0015)
    # 적절한 수렴 : (0.0015) --> for step in np.arange(10):
    # 반복 연산복잡도를 줄이는 중요한 역할을 하는 학습률

    train = optimizer.minimize(loss)

    init = tf.initialize_all_variables()
    sess = tf.Session()
    sess.run(init)


    for step in np.arange(10):
        sess.run(train)
        print(step, sess.run(W), sess.run(b))
        print(step, sess.run(loss))
        plt.plot(x_data, y_data, 'ro')
        plt.plot(x_data, sess.run(W) * x_data + sess.run(b))
        plt.xlabel('x')
        plt.ylabel('y')
        #plt.legend()
        plt.show()
//  [python]
def Data_Learning(x_data, y_data):
    W = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
    b = tf.Variable(tf.zeros([1]))

    y = W * x_data + b
    loss = tf.reduce_mean(tf.square(y - y_data))
    optimizer = tf.train.GradientDescentOptimizer(0.0015)
    train = optimizer.minimize(loss)

    init = tf.global_variables_initializer()
    sess = tf.Session()
    sess.run(init)


    for step in np.arange(10):
        sess.run(train)
        print(step, sess.run(W), sess.run(b))
        print(step, sess.run(loss))
        plt.plot(x_data, y_data, 'ro')
        plt.plot(x_data, sess.run(W) * x_data + sess.run(b))
        plt.xlabel('x')
        plt.ylabel('y')
        #plt.legend()
        plt.show()
//  [python]
def tfrun(tensor):
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        return sess.run(tensor)
//  [python]
def tfrun(tensor):
    with tf.compat.v1.Session() as sess:
        sess.run(tf.compat.v1.global_variables_initializer())
        return sess.run(tensor)
//  [python]
x_batch = np.random.rand(m)
y_batch = np.random.rand(1)

weights = np.random.rand(m)
biases = np.random.rand(m)

with tf.Session() as sess:

    x = tf.placeholder(tf.float32, shape=(m, ), name='x')
    y = tf.placeholder(tf.float32, shape=(1, ), name='y')
    # w = tf.Variable(np.random.rand(m), name='W', dtype=tf.float32)
    # b = tf.Variable(np.random.rand(m), name='b', dtype=tf.float32)
    w = tf.placeholder(tf.float32, shape=(m, ), name='W')
    b = tf.placeholder(tf.float32, shape=(m, ), name='b')

    mu = tf.constant(1, dtype=tf.float32)

    _ = tf.Variable(initial_value=np.random.rand(1))


    h = tf.reduce_sum(tf.multiply(w, x))
    c = tf.multiply(y, h)
    distances = tf.subtract(1., c)
    # maximum = tf.maximum(0., distances)
    #maximum = tf.boolean_mask(distances, tf.greater(0., distances))

    # Look here for gradient of SVM objective function: http://u.cs.biu.ac.il/~jkeshet/teaching/aml2016/sgd_optimization.pdf
    maximum = tf.cast(tf.greater(distances, 0.), tf.float32)

    g = tf.multiply(maximum, x)

    g = tf.multiply(mu, g)
    w = tf.subtract(w, g, name='update')

    sess.run(tf.initialize_all_variables())
    feed_dict = {x: x_batch, y: y_batch, w: weights, b: biases}
    sess.run(w, feed_dict)
    tf.train.Saver().save(sess, 'model.ckpt')
//  [python]
x_batch = np.random.rand(m)
y_batch = np.random.rand(1)

weights = np.random.rand(m)
biases = np.random.rand(m)

with tf.Session() as sess:

    x = tf.placeholder(tf.float32, shape=(m, ), name='x')
    y = tf.placeholder(tf.float32, shape=(1, ), name='y')
    # w = tf.Variable(np.random.rand(m), name='W', dtype=tf.float32)
    # b = tf.Variable(np.random.rand(m), name='b', dtype=tf.float32)
    w = tf.placeholder(tf.float32, shape=(m, ), name='W')
    b = tf.placeholder(tf.float32, shape=(m, ), name='b')

    mu = tf.constant(1, dtype=tf.float32)

    _ = tf.Variable(initial_value=np.random.rand(1))


    h = tf.reduce_sum(tf.multiply(w, x))
    c = tf.multiply(y, h)
    distances = tf.subtract(1., c)
    # maximum = tf.maximum(0., distances)
    #maximum = tf.boolean_mask(distances, tf.greater(0., distances))

    # Look here for gradient of SVM objective function: http://u.cs.biu.ac.il/~jkeshet/teaching/aml2016/sgd_optimization.pdf
    maximum = tf.cast(tf.greater(distances, 0.), tf.float32)

    g = tf.multiply(maximum, x)

    g = tf.multiply(mu, g)
    w = tf.subtract(w, g, name='update')

    sess.run(tf.global_variables_initializer())
    feed_dict = {x: x_batch, y: y_batch, w: weights, b: biases}
    sess.run(w, feed_dict)
    tf.train.Saver().save(sess, 'model.ckpt')
//  [python]
def elmo_vectors(z):
  embeddings = elmo(z, signature="default", as_dict=True)["elmo"]
  sess = tf.Session()
  sess.run(tf.initialize_all_variables())
  output = sess.run(tf.reduce_mean(embeddings,1))
  return output
//  [python]
def elmo_vectors(z):
  embeddings = elmo(z, signature="default", as_dict=True)["elmo"]
  sess = tf.compat.v1.Session()
  sess.run(tf.compat.v1.global_variables_initializer())
  output = sess.run(tf.reduce_mean(embeddings,1))
  return output
//  [python]
def train(session, graph_ops, num_actions, saver):
    """
    Train a model.
    """

    # Set up game environments (one per thread)
    envs = [gym.make(game) for i in range(n_threads)]

    summary_ops = build_summaries()
    summary_op = summary_ops[-1]

    # Initialize variables
    session.run(tf.initialize_all_variables())
    writer = writer_summary(summary_dir + "/qlearning", session.graph)

    # Initialize target network weights
    session.run(graph_ops["reset_target_network_params"])

    # Start n_threads actor-learner training threads
    actor_learner_threads = \
        [threading.Thread(target=actor_learner_thread,
                          args=(thread_id, envs[thread_id], session,
                                graph_ops, num_actions, summary_ops, saver))
         for thread_id in range(n_threads)]
    for t in actor_learner_threads:
        t.start()
        time.sleep(0.01)

    # Show the agents training and write summary statistics
    last_summary_time = 0
    while True:
        if show_training:
            for env in envs:
                env.render()
        now = time.time()
        if now - last_summary_time > summary_interval:
            summary_str = session.run(summary_op)
            writer.add_summary(summary_str, float(T))
            last_summary_time = now
    for t in actor_learner_threads:
        t.join()
//  [python]
def train(session, graph_ops, num_actions, saver):
    """
    Train a model.
    """

    # Set up game environments (one per thread)
    envs = [gym.make(game) for i in range(n_threads)]

    summary_ops = build_summaries()
    summary_op = summary_ops[-1]

    # Initialize variables
    session.run(tf.global_variables_initializer())
    writer = writer_summary(summary_dir + "/qlearning", session.graph)

    # Initialize target network weights
    session.run(graph_ops["reset_target_network_params"])

    # Start n_threads actor-learner training threads
    actor_learner_threads = \
        [threading.Thread(target=actor_learner_thread,
                          args=(thread_id, envs[thread_id], session,
                                graph_ops, num_actions, summary_ops, saver))
         for thread_id in range(n_threads)]
    for t in actor_learner_threads:
        t.start()
        time.sleep(0.01)

    # Show the agents training and write summary statistics
    last_summary_time = 0
    while True:
        if show_training:
            for env in envs:
                env.render()
        now = time.time()
        if now - last_summary_time > summary_interval:
            summary_str = session.run(summary_op)
            writer.add_summary(summary_str, float(T))
            last_summary_time = now
    for t in actor_learner_threads:
        t.join()
//  [python]
def __call__(self, batcher, placeholders, loss, session=None):
        self.loss = loss
        minimization_op = self.optimizer.minimize(loss)

        if session is None:
            session = tf.Session()

        init = tf.initialize_all_variables()
        session.run(init)
        epoch = 1
        while epoch < self.max_epochs:
            iteration = 1
            total_loss = 0
            total = 0
            for values in batcher:
                iteration += 1
                total += len(values[-1])
                feed_dict = {}
                for i in range(0, len(placeholders)):
                    feed_dict[placeholders[i]] = values[i]
                _, current_loss = session.run([minimization_op, loss], feed_dict=feed_dict)
                total_loss += current_loss

            print("Epoch ", str(epoch), "\tLoss ", total_loss)
            epoch += 1

        return session
//  [python]
def __call__(self, batcher, placeholders, loss, session=None):
        self.loss = loss
        minimization_op = self.optimizer.minimize(loss)

        if session is None:
            session = tf.Session()

        init = tf.global_variables_initializer()
        session.run(init)
        epoch = 1
        while epoch < self.max_epochs:
            iteration = 1
            total_loss = 0
            total = 0
            for values in batcher:
                iteration += 1
                total += len(values[-1])
                feed_dict = {}
                for i in range(0, len(placeholders)):
                    feed_dict[placeholders[i]] = values[i]
                _, current_loss = session.run([minimization_op, loss], feed_dict=feed_dict)
                total_loss += current_loss

            print("Epoch ", str(epoch), "\tLoss ", total_loss)
            epoch += 1

        return session
//  [python]
def __init__(self, sess, state_size, action_size, BATCH_SIZE, TAU, LEARNING_RATE):
        self.sess = sess
        self.BATCH_SIZE = BATCH_SIZE
        self.TAU = TAU
        self.LEARNING_RATE = LEARNING_RATE

        tf.compat.v1.keras.backend.set_session(sess)

        #Now create the model
        self.model , self.weights, self.state = self.create_actor_network(state_size, action_size)   
        self.target_model, self.target_weights, self.target_state = self.create_actor_network(state_size, action_size) 
        self.action_gradient = tf.placeholder(tf.float32,[None, action_size])
        self.params_grad = tf.gradients(self.model.output, self.weights, -self.action_gradient)
        grads = zip(self.params_grad, self.weights)
        self.optimize = tf.train.AdamOptimizer(LEARNING_RATE).apply_gradients(grads)
        self.sess.run(tf.initialize_all_variables())
//  [python]
def __init__(self, sess, state_size, action_size, BATCH_SIZE, TAU, LEARNING_RATE):
        self.sess = sess
        self.BATCH_SIZE = BATCH_SIZE
        self.TAU = TAU
        self.LEARNING_RATE = LEARNING_RATE

        tf.compat.v1.keras.backend.set_session(sess)

        #Now create the model
        self.model , self.weights, self.state = self.create_actor_network(state_size, action_size)   
        self.target_model, self.target_weights, self.target_state = self.create_actor_network(state_size, action_size) 
        self.action_gradient = tf.placeholder(tf.float32,[None, action_size])
        self.params_grad = tf.gradients(self.model.output, self.weights, -self.action_gradient)
        grads = zip(self.params_grad, self.weights)
        self.optimize = tf.train.AdamOptimizer(LEARNING_RATE).apply_gradients(grads)
        self.sess.run(tf.compat.v1.global_variables_initializer())
//  [python]
def __init__(self, sess, state_size, action_size, BATCH_SIZE, TAU, LEARNING_RATE):
        self.sess = sess
        self.BATCH_SIZE = BATCH_SIZE
        self.TAU = TAU
        self.LEARNING_RATE = LEARNING_RATE
        self.action_size = action_size
        
        tf.compat.v1.keras.backend.set_session(sess)

        #Now create the model
        self.model, self.action, self.state = self.create_critic_network(state_size, action_size)  
        self.target_model, self.target_action, self.target_state = self.create_critic_network(state_size, action_size)  
        self.action_grads = tf.gradients(self.model.output, self.action)  #GRADIENTS for policy update
        self.sess.run(tf.initialize_all_variables())
//  [python]
def __init__(self, sess, state_size, action_size, BATCH_SIZE, TAU, LEARNING_RATE):
        self.sess = sess
        self.BATCH_SIZE = BATCH_SIZE
        self.TAU = TAU
        self.LEARNING_RATE = LEARNING_RATE
        self.action_size = action_size
        
        tf.compat.v1.keras.backend.set_session(sess)

        #Now create the model
        self.model, self.action, self.state = self.create_critic_network(state_size, action_size)  
        self.target_model, self.target_action, self.target_state = self.create_critic_network(state_size, action_size)  
        self.action_grads = tf.gradients(self.model.output, self.action)  #GRADIENTS for policy update
        self.sess.run(tf.compat.v1.global_variables_initializer())
//  [python]
keep_prob = tf.placeholder(tf.float32)

## conv1 layer ##

## conv2 layer ##

## func1 layer ##

## func2 layer ##


# the error between prediction and real data
cross_entropy = tf.reduce_mean(-tf.reduce_sum(ys * tf.log(prediction),
                                              reduction_indices=[1]))       # loss
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

sess = tf.Session()
# important step
# tf.initialize_all_variables() no long valid from
# 2017-03-02 if using tensorflow >= 0.12
if int((tf.__version__).split('.')[1]) < 12 and int((tf.__version__).split('.')[0]) < 1:
    init = tf.initialize_all_variables()
else:
    init = tf.global_variables_initializer()
//  [python]
keep_prob = tf.placeholder(tf.float32)

## conv1 layer ##

## conv2 layer ##

## func1 layer ##

## func2 layer ##


# the error between prediction and real data
cross_entropy = tf.reduce_mean(-tf.reduce_sum(ys * tf.log(prediction),
                                              reduction_indices=[1]))       # loss
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

sess = tf.Session()
# important step
# tf.initialize_all_variables() no long valid from
# 2017-03-02 if using tensorflow >= 0.12
init = tf.global_variables_initializer()
//  [python]
def testLossWithL2Regularization(self, agent_class):
    q_net = DummyNet(
        self._observation_spec, self._action_spec, l2_regularization_weight=1.0
    )
    agent = agent_class(
        self._time_step_spec, self._action_spec, q_network=q_net, optimizer=None
    )

    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)

    actions = tf.constant([0, 1], dtype=tf.int32)
    action_steps = policy_step.PolicyStep(actions)

    rewards = tf.constant([10, 20], dtype=tf.float32)
    discounts = tf.constant([0.9, 0.9], dtype=tf.float32)
    next_observations = tf.constant([[5, 6], [7, 8]], dtype=tf.float32)
    next_time_steps = ts.transition(next_observations, rewards, discounts)

    experience = trajectories_test_utils.stacked_trajectory_from_transition(
        time_steps, action_steps, next_time_steps
    )

    # See the loss explanation in testLoss above.
    # L2_regularization_loss: 2^2 + 1^2 + 1^2 + 1^2 = 7.0
    # Overall loss: 26.0 (from testLoss) + 7.0 = 33.0
    expected_loss = 33.0
    loss, _ = agent._loss(experience)

    self.evaluate(tf.compat.v1.initialize_all_variables())
    self.assertAllClose(self.evaluate(loss), expected_loss)
//  [python]
def testLossWithL2Regularization(self, agent_class):
    q_net = DummyNet(
        self._observation_spec, self._action_spec, l2_regularization_weight=1.0
    )
    agent = agent_class(
        self._time_step_spec, self._action_spec, q_network=q_net, optimizer=None
    )

    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)

    actions = tf.constant([0, 1], dtype=tf.int32)
    action_steps = policy_step.PolicyStep(actions)

    rewards = tf.constant([10, 20], dtype=tf.float32)
    discounts = tf.constant([0.9, 0.9], dtype=tf.float32)
    next_observations = tf.constant([[5, 6], [7, 8]], dtype=tf.float32)
    next_time_steps = ts.transition(next_observations, rewards, discounts)

    experience = trajectories_test_utils.stacked_trajectory_from_transition(
        time_steps, action_steps, next_time_steps
    )

    # See the loss explanation in testLoss above.
    # L2_regularization_loss: 2^2 + 1^2 + 1^2 + 1^2 = 7.0
    # Overall loss: 26.0 (from testLoss) + 7.0 = 33.0
    expected_loss = 33.0
    loss, _ = agent._loss(experience)

    self.evaluate(tf.compat.v1.global_variables_initializer())
    self.assertAllClose(self.evaluate(loss), expected_loss)
//  [python]
def initialize_global_variables(sess=None):
    """Excute ``sess.run(tf.global_variables_initializer())`` for TF12+ or
    sess.run(tf.initialize_all_variables()) for TF11.

    Parameters
    ----------
    sess : a Session
    """
    assert sess is not None
    try:    # TF12
        sess.run(tf.global_variables_initializer())
    except: # TF11
        sess.run(tf.initialize_all_variables())
//  [python]
def initialize_global_variables(sess=None):
    """Excute ``sess.run(tf.global_variables_initializer())`` for TF12+ or
    sess.run(tf.initialize_all_variables()) for TF11.

    Parameters
    ----------
    sess : a Session
    """
    assert sess is not None
    try:    # TF12
        sess.run(tf.compat.v1.global_variables_initializer())
    except: # TF11
        sess.run(tf.compat.v1.global_variables_initializer())
//  [python]
def build_graph(self):
    """Build the graph for the full model."""
    opts = self._options
    # The training data. A text file.
    (words, counts, words_per_epoch, self._epoch, self._words, examples,
     labels) = word2vec.skipgram_word2vec(filename=opts.train_data,
                                 batch_size=opts.batch_size,
                                 window_size=opts.window_size,
                                 min_count=opts.min_count,
                                 subsample=opts.subsample)
    (opts.vocab_words, opts.vocab_counts,
     opts.words_per_epoch) = self._session.run([words, counts, words_per_epoch])
    opts.vocab_size = len(opts.vocab_words)
    print("Data file: ", opts.train_data)
    print("Vocab size: ", opts.vocab_size - 1, " + UNK")
    print("Words per epoch: ", opts.words_per_epoch)
    self._examples = examples
    self._labels = labels
    self._id2word = opts.vocab_words
    for i, w in enumerate(self._id2word):
      self._word2id[w] = i
    loss = self.calculate_loss(examples, labels)
    self._loss = loss

    if opts.normclip:
      self._clip_ops = self.clip_ops_graph(self._examples, self._labels, self._neg_idxs)

    if opts.adagrad:
      print("Using Adagrad as an optimizer!")
      self.optimize_adagrad(loss)
    else:
      # Using Standard SGD
      self.optimize(loss)
    # Properly initialize all variables.
    self.check_op = tf.add_check_numerics_ops()

    tf.initialize_all_variables().run()

    try:
      print('Try using saver version v2')
      self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2, max_to_keep = opts.max_to_keep)
    except:
      print('Default to saver version v1')
      self.saver = tf.train.Saver(max_to_keep=opts.max_to_keep)
//  [python]
def build_graph(self):
    """Build the graph for the full model."""
    opts = self._options
    # The training data. A text file.
    (words, counts, words_per_epoch, self._epoch, self._words, examples,
     labels) = word2vec.skipgram_word2vec(filename=opts.train_data,
                                 batch_size=opts.batch_size,
                                 window_size=opts.window_size,
                                 min_count=opts.min_count,
                                 subsample=opts.subsample)
    (opts.vocab_words, opts.vocab_counts,
     opts.words_per_epoch) = self._session.run([words, counts, words_per_epoch])
    opts.vocab_size = len(opts.vocab_words)
    print("Data file: ", opts.train_data)
    print("Vocab size: ", opts.vocab_size - 1, " + UNK")
    print("Words per epoch: ", opts.words_per_epoch)
    self._examples = examples
    self._labels = labels
    self._id2word = opts.vocab_words
    for i, w in enumerate(self._id2word):
      self._word2id[w] = i
    loss = self.calculate_loss(examples, labels)
    self._loss = loss

    if opts.normclip:
      self._clip_ops = self.clip_ops_graph(self._examples, self._labels, self._neg_idxs)

    if opts.adagrad:
      print("Using Adagrad as an optimizer!")
      self.optimize_adagrad(loss)
    else:
      # Using Standard SGD
      self.optimize(loss)
    # Properly initialize all variables.
    self.check_op = tf.add_check_numerics_ops()

    tf.global_variables_initializer().run()

    try:
      print('Try using saver version v2')
      self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2, max_to_keep = opts.max_to_keep)
    except:
      print('Default to saver version v1')
      self.saver = tf.train.Saver(max_to_keep=opts.max_to_keep)
//  [python]
def Data_Learning(x_data, y_data):
    W = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
    # init = tf.initialize_all_variables()
    # sess = tf.Session()
    # sess.run(init)
    # sess.run(W)
    # print('sess.run(W)= ', sess.run(W))
    # array([0.05211711], dtype=float32)
    b = tf.Variable(tf.zeros([1]))
    # print('sess.run(b)= ', sess.run(b))
    # array([0.], dtype=float32)
    y = W * x_data + b
    # print('sess.run(y)= ', sess.run(y))
    # y=np.round(sess.run(y), 1)
    # print(y)
    loss = tf.reduce_mean(tf.square(y - y_data))
    # print(np.round(sess.run(loss),1))
    optimizer = tf.train.GradientDescentOptimizer(0.0015)
    train = optimizer.minimize(loss)
    init = tf.initialize_all_variables()
    sess = tf.Session()
    sess.run(init)

    train_set = []  ###
    for step in np.arange(10):
        sess.run(train)
        print(step, sess.run(W), sess.run(b))
        print(step, sess.run(loss))
        train_set.append([sess.run(W), sess.run(b), sess.run(loss)])  ###

        plt.plot(x_data, y_data, 'ro')
        plt.plot(x_data, sess.run(W) * x_data + sess.run(b))
        plt.xlabel('x')
        plt.ylabel('y')
        #plt.legend()
        plt.show()

    W_data = np.array([t[0] for t in train_set])
    v_data = np.array([t[1] for t in train_set])
    Loss_data= np.array([t[2] for t in train_set])

    return W_data,v_data, Loss_data
//  [python]
def Data_Learning(x_data, y_data):
    W = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
    b = tf.Variable(tf.zeros([1]))
    y = W * x_data + b
    loss = tf.reduce_mean(tf.square(y - y_data))
    optimizer = tf.train.GradientDescentOptimizer(0.0015)
    train = optimizer.minimize(loss)
    init = tf.global_variables_initializer()
    sess = tf.Session()
    sess.run(init)

    train_set = []  ###
    for step in np.arange(10):
        sess.run(train)
        print(step, sess.run(W), sess.run(b))
        print(step, sess.run(loss))
        train_set.append([sess.run(W), sess.run(b), sess.run(loss)])  ###

        plt.plot(x_data, y_data, 'ro')
        plt.plot(x_data, sess.run(W) * x_data + sess.run(b))
        plt.xlabel('x')
        plt.ylabel('y')
        #plt.legend()
        plt.show()

    W_data = np.array([t[0] for t in train_set])
    v_data = np.array([t[1] for t in train_set])
    Loss_data= np.array([t[2] for t in train_set])

    return W_data,v_data, Loss_data
//  [python]
def test_sample_auxiliary_op(self):
    p_fn, q_fn = sampling.mean_field_fn()
    p = p_fn(tf.float32, (), 'test_prior', True, tf.get_variable).distribution
    q = q_fn(tf.float32, (), 'test_posterior', True,
             tf.get_variable).distribution

    # Test benign auxiliary variable
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 1e-10)
    session_config = tf.ConfigProto(
        graph_options=tf.GraphOptions(
            rewrite_options=rewriter_config_pb2.RewriterConfig(
                arithmetic_optimization=rewriter_config_pb2.RewriterConfig
                .AGGRESSIVE)))

    sess = tf.Session(config=session_config)
    sess.run(tf.initialize_all_variables())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1.1, session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(0.5), session=sess)
    print(sess.run(q.scale))

    sess.run(sample_op)

    tolerance = 0.0001
    self.assertLess(np.abs(sess.run(p.scale) - 1.), tolerance)
    self.assertLess(np.abs(sess.run(p.loc) - 1.), tolerance)
    self.assertLess(np.abs(sess.run(q.scale) - 0.5), tolerance)
    self.assertLess(np.abs(sess.run(q.loc) - 1.1), tolerance)

    # Test fully determining auxiliary variable
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 1. - 1e-10)
    sess.run(tf.initialize_all_variables())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1.1, session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(.5), session=sess)

    sess.run(sample_op)

    self.assertLess(np.abs(sess.run(q.loc) - sess.run(p.loc)), tolerance)
    self.assertLess(sess.run(p.scale), tolerance)
    self.assertLess(sess.run(q.scale), tolerance)

    # Test delta posterior
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 0.5)
    sess.run(tf.initialize_all_variables())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1.1, session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(1e-10), session=sess)

    sess.run(sample_op)

    self.assertLess(np.abs(sess.run(q.loc) - 1.1), tolerance)
    self.assertLess(sess.run(q.scale), tolerance)

    # Test prior is posterior
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 0.5)
    sess.run(tf.initialize_all_variables())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1., session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)

    sess.run(sample_op)

    self.assertLess(np.abs(sess.run(q.loc - p.loc)), tolerance)
    self.assertLess(np.abs(sess.run(q.scale - p.scale)), tolerance)
//  [python]
def test_sample_auxiliary_op(self):
    p_fn, q_fn = sampling.mean_field_fn()
    p = p_fn(tf.float32, (), 'test_prior', True, tf.get_variable).distribution
    q = q_fn(tf.float32, (), 'test_posterior', True,
             tf.get_variable).distribution

    # Test benign auxiliary variable
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 1e-10)
    session_config = tf.ConfigProto(
        graph_options=tf.GraphOptions(
            rewrite_options=rewriter_config_pb2.RewriterConfig(
                arithmetic_optimization=rewriter_config_pb2.RewriterConfig
                .AGGRESSIVE)))

    sess = tf.Session(config=session_config)
    sess.run(tf.global_variables_initializer())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1.1, session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(0.5), session=sess)
    print(sess.run(q.scale))

    sess.run(sample_op)

    tolerance = 0.0001
    self.assertLess(np.abs(sess.run(p.scale) - 1.), tolerance)
    self.assertLess(np.abs(sess.run(p.loc) - 1.), tolerance)
    self.assertLess(np.abs(sess.run(q.scale) - 0.5), tolerance)
    self.assertLess(np.abs(sess.run(q.loc) - 1.1), tolerance)

    # Test fully determining auxiliary variable
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 1. - 1e-10)
    sess.run(tf.global_variables_initializer())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1.1, session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(.5), session=sess)

    sess.run(sample_op)

    self.assertLess(np.abs(sess.run(q.loc) - sess.run(p.loc)), tolerance)
    self.assertLess(sess.run(p.scale), tolerance)
    self.assertLess(sess.run(q.scale), tolerance)

    # Test delta posterior
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 0.5)
    sess.run(tf.global_variables_initializer())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1.1, session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(1e-10), session=sess)

    sess.run(sample_op)

    self.assertLess(np.abs(sess.run(q.loc) - 1.1), tolerance)
    self.assertLess(sess.run(q.scale), tolerance)

    # Test prior is posterior
    sample_op, _ = sampling.sample_auxiliary_op(p, q, 0.5)
    sess.run(tf.global_variables_initializer())
    p.loc.load(1., session=sess)
    p.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)
    q.loc.load(1., session=sess)
    q.untransformed_scale.load(self._softplus_inverse_np(1.), session=sess)

    sess.run(sample_op)

    self.assertLess(np.abs(sess.run(q.loc - p.loc)), tolerance)
    self.assertLess(np.abs(sess.run(q.scale - p.scale)), tolerance)
//  [python]
def testLossWithL2Regularization(self, agent_class):
    q_net = DummyNet(self._observation_spec, self._action_spec,
                     l2_regularization_weight=1.0)
    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        q_network=q_net,
        optimizer=None)

    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)

    actions = tf.constant([0, 1], dtype=tf.int32)
    action_steps = policy_step.PolicyStep(actions)

    rewards = tf.constant([10, 20], dtype=tf.float32)
    discounts = tf.constant([0.9, 0.9], dtype=tf.float32)
    next_observations = tf.constant([[5, 6], [7, 8]], dtype=tf.float32)
    next_time_steps = ts.transition(next_observations, rewards, discounts)

    experience = trajectories_test_utils.stacked_trajectory_from_transition(
        time_steps, action_steps, next_time_steps)

    # See the loss explanation in testLoss above.
    # L2_regularization_loss: 2^2 + 1^2 + 1^2 + 1^2 = 7.0
    # Overall loss: 26.0 (from testLoss) + 7.0 = 33.0
    expected_loss = 33.0
    loss, _ = agent._loss(experience)

    self.evaluate(tf.compat.v1.initialize_all_variables())
    self.assertAllClose(self.evaluate(loss), expected_loss)
//  [python]
def testLossWithL2Regularization(self, agent_class):
    q_net = DummyNet(self._observation_spec, self._action_spec,
                     l2_regularization_weight=1.0)
    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        q_network=q_net,
        optimizer=None)

    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)

    actions = tf.constant([0, 1], dtype=tf.int32)
    action_steps = policy_step.PolicyStep(actions)

    rewards = tf.constant([10, 20], dtype=tf.float32)
    discounts = tf.constant([0.9, 0.9], dtype=tf.float32)
    next_observations = tf.constant([[5, 6], [7, 8]], dtype=tf.float32)
    next_time_steps = ts.transition(next_observations, rewards, discounts)

    experience = trajectories_test_utils.stacked_trajectory_from_transition(
        time_steps, action_steps, next_time_steps)

    # See the loss explanation in testLoss above.
    # L2_regularization_loss: 2^2 + 1^2 + 1^2 + 1^2 = 7.0
    # Overall loss: 26.0 (from testLoss) + 7.0 = 33.0
    expected_loss = 33.0
    loss, _ = agent._loss(experience)

    self.evaluate(tf.compat.v1.global_variables_initializer())
    self.assertAllClose(self.evaluate(loss), expected_loss)
//  [python]
def test_encoder(self, encoder_f):
    minibatch = np.ones(shape=(10, 64, 64, 1), dtype=np.float32)
    input_tensor = tf.placeholder(tf.float32, shape=(None, 64, 64, 1))
    latent_mean, latent_logvar = encoder_f(input_tensor, 10)
    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      sess.run(
          [latent_mean, latent_logvar], feed_dict={input_tensor: minibatch})
//  [python]
def test_encoder(self, encoder_f):
    minibatch = np.ones(shape=(10, 64, 64, 1), dtype=np.float32)
    input_tensor = tf.placeholder(tf.float32, shape=(None, 64, 64, 1))
    latent_mean, latent_logvar = encoder_f(input_tensor, 10)
    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(
          [latent_mean, latent_logvar], feed_dict={input_tensor: minibatch})