jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_train_incremental_pca(self):
        iris = datasets.load_iris()
        iris = iris.data[: BATCH_SIZE * BATCHES]
        pca = decomposition.IncrementalPCA(
            n_components=COMPONENTS, batch_size=BATCH_SIZE
        )
        pca.fit(iris)
        proj_ref = pca.transform(iris)
        batch = tf.placeholder(dtype=tf.float32, shape=[BATCH_SIZE, FEATURES])
        step = tf.train.get_or_create_global_step()
        train_op, _ = train_incremental_pca(step, batch, COMPONENTS)
        with self.test_session() as sess:
            sess.run(tf.initialize_all_variables())
            for iris_batch in np.split(iris, BATCHES):
                sess.run(train_op, feed_dict={batch: iris_batch})
            proj = sess.run(incremental_pca(iris, COMPONENTS))
        self.assertAllClose(proj, proj_ref)
//  [python]
def test_train_incremental_pca(self):
        iris = datasets.load_iris()
        iris = iris.data[: BATCH_SIZE * BATCHES]
        pca = decomposition.IncrementalPCA(
            n_components=COMPONENTS, batch_size=BATCH_SIZE
        )
        pca.fit(iris)
        proj_ref = pca.transform(iris)
        batch = tf.placeholder(dtype=tf.float32, shape=[BATCH_SIZE, FEATURES])
        step = tf.train.get_or_create_global_step()
        train_op, _ = train_incremental_pca(step, batch, COMPONENTS)
        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            for iris_batch in np.split(iris, BATCHES):
                sess.run(train_op, feed_dict={batch: iris_batch})
            proj = sess.run(incremental_pca(iris, COMPONENTS))
        self.assertAllClose(proj, proj_ref)
//  [python]
def test_decoder(self, decoder_f):
    latent_variable = np.ones(shape=(10, 15), dtype=np.float32)
    input_tensor = tf.placeholder(tf.float32, shape=(None, 15))
    images = decoder_f(input_tensor, [64, 64, 1])
    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      sess.run(images, feed_dict={input_tensor: latent_variable})
//  [python]
def test_decoder(self, decoder_f):
    latent_variable = np.ones(shape=(10, 15), dtype=np.float32)
    input_tensor = tf.placeholder(tf.float32, shape=(None, 15))
    images = decoder_f(input_tensor, [64, 64, 1])
    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(images, feed_dict={input_tensor: latent_variable})
//  [python]
def test_discriminator(self, discriminator_f):
    images = np.ones(shape=(32, 10), dtype=np.float32)
    input_tensor = tf.placeholder(tf.float32, shape=(None, 10))
    logits, probs = discriminator_f(input_tensor)
    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      sess.run([logits, probs], feed_dict={input_tensor: images})
//  [python]
def test_discriminator(self, discriminator_f):
    images = np.ones(shape=(32, 10), dtype=np.float32)
    input_tensor = tf.placeholder(tf.float32, shape=(None, 10))
    logits, probs = discriminator_f(input_tensor)
    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run([logits, probs], feed_dict={input_tensor: images})
//  [python]
d = tf.add(c, v)
global_step = tf.train.get_or_create_global_step()
global_step_inc = tf.assign_add(global_step, 1)
hooks = [tf.train.StopAtStepHook(last_step=2)]
with tf.Session() as mon_sess:
    mon_sess.run(tf.initialize_all_variables())
    for i in range(2):
        print(mon_sess.run([d, global_step_inc], feed_dict={a: [1.0, 2.0, 3.0], b: [1.0, 2.0, 3.0]}))

    signatures = {
        tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: {
            'inputs': {'a': a, "b": b},
            'outputs': {"d": d},
            'method_name': tf.saved_model.signature_constants.PREDICT_METHOD_NAME
        }
    }

    export_dir = os.path.dirname(os.path.abspath(__file__)) + "../../../../target/test-classes/export"
    try:
        tf.gfile.DeleteRecursively(export_dir)
    except tf.errors.OpError:
        pass
    builder = tf.saved_model.builder.SavedModelBuilder(export_dir)

    signature_def_map = {}
    for key, sig in signatures.items():
        signature_def_map[key] = tf.saved_model.signature_def_utils.build_signature_def(
            inputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                    sig['inputs'].items()},
            outputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                     sig['outputs'].items()},
            method_name=sig['method_name'] if 'method_name' in sig else key)

        builder.add_meta_graph_and_variables(
            mon_sess,
            tf.saved_model.tag_constants.SERVING.split(','),
            signature_def_map=signature_def_map,
            clear_devices=True)
        builder.save()
//  [python]
d = tf.add(c, v)
global_step = tf.train.get_or_create_global_step()
global_step_inc = tf.assign_add(global_step, 1)
hooks = [tf.train.StopAtStepHook(last_step=2)]
with tf.Session() as mon_sess:
    mon_sess.run(tf.global_variables_initializer())
    for i in range(2):
        print(mon_sess.run([d, global_step_inc], feed_dict={a: [1.0, 2.0, 3.0], b: [1.0, 2.0, 3.0]}))

    signatures = {
        tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: {
            'inputs': {'a': a, "b": b},
            'outputs': {"d": d},
            'method_name': tf.saved_model.signature_constants.PREDICT_METHOD_NAME
        }
    }

    export_dir = os.path.dirname(os.path.abspath(__file__)) + "../../../../target/test-classes/export"
    try:
        tf.gfile.DeleteRecursively(export_dir)
    except tf.errors.OpError:
        pass
    builder = tf.saved_model.builder.SavedModelBuilder(export_dir)

    signature_def_map = {}
    for key, sig in signatures.items():
        signature_def_map[key] = tf.saved_model.signature_def_utils.build_signature_def(
            inputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                    sig['inputs'].items()},
            outputs={name: tf.saved_model.utils.build_tensor_info(tensor) for name, tensor in
                     sig['outputs'].items()},
            method_name=sig['method_name'] if 'method_name' in sig else key)

        builder.add_meta_graph_and_variables(
            mon_sess,
            tf.saved_model.tag_constants.SERVING.split(','),
            signature_def_map=signature_def_map,
            clear_devices=True)
        builder.save()
//  [python]
def create_train_op(self, X, Y, alpha):
        # load params
        opt_params = self.opt_params
        print('creating train_op with params:', opt_params)

        # create loss
        self.loss = self.create_objective(X, Y, opt_params)

        # create params
        params = self.get_params()

        # create optimizer
        self.optimizer = self.create_optimzier(opt_params)

        # create gradients
        grads = self.create_gradients(self.loss, params)

        # create training op
        with tf.compat.v1.name_scope('optimizer'):
            train_op = self.create_updates(params, grads, alpha, opt_params)

        # initialize the optimizer variabLes
        optimizer_vars = [v for v in tf.compat.v1.global_variables() if 'optimizer/' in v.name
                          or 'Adam' in v.name]
        #init = tf.variables_initializer(optimizer_vars)
        init = tf.compat.v1.initialize_all_variables()
        self.sess.run(init)

        return train_op
//  [python]
def create_train_op(self, X, Y, alpha):
        # load params
        opt_params = self.opt_params
        print('creating train_op with params:', opt_params)

        # create loss
        self.loss = self.create_objective(X, Y, opt_params)

        # create params
        params = self.get_params()

        # create optimizer
        self.optimizer = self.create_optimzier(opt_params)

        # create gradients
        grads = self.create_gradients(self.loss, params)

        # create training op
        with tf.compat.v1.name_scope('optimizer'):
            train_op = self.create_updates(params, grads, alpha, opt_params)

        # initialize the optimizer variabLes
        optimizer_vars = [v for v in tf.compat.v1.global_variables() if 'optimizer/' in v.name
                          or 'Adam' in v.name]
        #init = tf.variables_initializer(optimizer_vars)
        init = tf.compat.v1.global_variables_initializer()
        self.sess.run(init)

        return train_op
//  [python]
def testPolicy(self):
    q_net = critic_network.CriticNetwork(
        (self._observation_spec, self._action_spec)
    )
    agent = qtopt_agent.QtOptAgent(
        self._time_step_spec,
        self._action_spec,
        q_network=q_net,
        optimizer=None,
        init_mean_cem=self._mean,
        init_var_cem=self._var,
        num_samples_cem=self._num_samples,
        actions_sampler=self._sampler,
    )
    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)
    policy = agent.policy
    action_step = policy.action(time_steps)
    # Batch size 2.
    self.assertAllEqual(
        [2] + self._action_spec.shape.as_list(),
        action_step.action.shape,
    )
    self.evaluate(tf.compat.v1.initialize_all_variables())
    actions_ = self.evaluate(action_step.action)
    self.assertTrue(all(actions_ <= self._action_spec.maximum))
    self.assertTrue(all(actions_ >= self._action_spec.minimum))
//  [python]
def testPolicy(self):
    q_net = critic_network.CriticNetwork(
        (self._observation_spec, self._action_spec)
    )
    agent = qtopt_agent.QtOptAgent(
        self._time_step_spec,
        self._action_spec,
        q_network=q_net,
        optimizer=None,
        init_mean_cem=self._mean,
        init_var_cem=self._var,
        num_samples_cem=self._num_samples,
        actions_sampler=self._sampler,
    )
    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)
    policy = agent.policy
    action_step = policy.action(time_steps)
    # Batch size 2.
    self.assertAllEqual(
        [2] + self._action_spec.shape.as_list(),
        action_step.action.shape,
    )
    self.evaluate(tf.compat.v1.global_variables_initializer())
    actions_ = self.evaluate(action_step.action)
    self.assertTrue(all(actions_ <= self._action_spec.maximum))
    self.assertTrue(all(actions_ >= self._action_spec.minimum))
//  [python]
def testPolicy(self):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    agent = greedy_agent.GreedyRewardPredictionAgent(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=None,
        num_samples_list=self._num_samples_list,
    )
    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)
    policy = agent.policy
    action_step = policy.action(time_steps)
    # Batch size 2.
    self.assertAllEqual([2], action_step.action.shape)
    self.evaluate(tf.compat.v1.initialize_all_variables())
    actions = self.evaluate(action_step.action)
    self.assertAllEqual(actions, [1, 2])
//  [python]
def testPolicy(self):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    agent = greedy_agent.GreedyRewardPredictionAgent(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=None,
        num_samples_list=self._num_samples_list,
    )
    observations = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
    time_steps = ts.restart(observations, batch_size=2)
    policy = agent.policy
    action_step = policy.action(time_steps)
    # Batch size 2.
    self.assertAllEqual([2], action_step.action.shape)
    self.evaluate(tf.compat.v1.global_variables_initializer())
    actions = self.evaluate(action_step.action)
    self.assertAllEqual(actions, [1, 2])
//  [python]
def Data_Learning(x_data, y_data):
    W = tf.Variable(tf.random_uniform([1], -1.0, 1.0))
    # init = tf.initialize_all_variables()
    # sess = tf.Session()
    # sess.run(init)
    # sess.run(W)
    # print('sess.run(W)= ', sess.run(W))
    # array([0.05211711], dtype=float32)
    b = tf.Variable(tf.zeros([1]))
    # sess.run(b)
    # array([0.], dtype=float32)
    y = W * x_data + b
    # sess.run(y)
    # print(np.round(sess.run(y),1))
    loss = tf.reduce_mean(tf.square(y - y_data))
    # print(np.round(sess.run(loss),1))
    #lo.append(loss)
    optimizer = tf.train.GradientDescentOptimizer(0.0015)
    train = optimizer.minimize(loss)
    init = tf.initialize_all_variables()
    sess = tf.Session()
    sess.run(init)

    train_set = []  ###
    for step in np.arange(10):
        sess.run(train)
        print(step, sess.run(W), sess.run(b))
        print(step, sess.run(loss))
        train_set.append([sess.run(W), sess.run(b), sess.run(loss)])  ###

        plt.plot(x_data, y_data, 'ro')
        plt.plot(x_data, sess.run(W) * x_data + sess.run(b))
        plt.xlabel('x')
        plt.ylabel('y')
        #plt.legend()
        plt.show()

    W_data = np.array([t[0] for t in train_set])
    v_data = np.array([t[1] for t in train_set])
    Loss_data= np.array([t[2] for t in train_set])

    return W_data,v_data, Loss_data
//  [python]
def testLoss(self, agent_class, use_num_samples_list):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps_nested_rewards(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)

    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )

    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=None,
    )
    init_op = agent.initialize()
    if not tf.executing_eagerly():
      with self.cached_session() as sess:
        common.initialize_uninitialized_variables(sess)
        self.assertIsNone(sess.run(init_op))
    loss, _ = agent._loss(experience)
    self.evaluate(tf.compat.v1.initialize_all_variables())
    self.assertAllClose(self.evaluate(loss), 42.25)
//  [python]
def testLoss(self, agent_class, use_num_samples_list):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps_nested_rewards(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)

    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )

    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=None,
    )
    init_op = agent.initialize()
    if not tf.executing_eagerly():
      with self.cached_session() as sess:
        common.initialize_uninitialized_variables(sess)
        self.assertIsNone(sess.run(init_op))
    loss, _ = agent._loss(experience)
    self.evaluate(tf.compat.v1.global_variables_initializer())
    self.assertAllClose(self.evaluate(loss), 42.25)
//  [python]
def testTrainAgent(self, agent_class, use_num_samples_list):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )
    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
    )
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    loss_before, _ = agent.train(experience, None)
    loss_after, _ = agent.train(experience, None)
    self.evaluate(tf.compat.v1.initialize_all_variables())
    self.assertAllClose(self.evaluate(loss_before), 42.25)
    self.assertAllClose(self.evaluate(loss_after), 93.46)
//  [python]
def testTrainAgent(self, agent_class, use_num_samples_list):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )
    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
    )
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    loss_before, _ = agent.train(experience, None)
    loss_after, _ = agent.train(experience, None)
    self.evaluate(tf.compat.v1.global_variables_initializer())
    self.assertAllClose(self.evaluate(loss_before), 42.25)
    self.assertAllClose(self.evaluate(loss_after), 93.46)
//  [python]
def testTrainAgentWithMask(self, agent_class, use_num_samples_list):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    time_step_spec = ts.time_step_spec((
        tensor_spec.TensorSpec([2], tf.float32),
        tensor_spec.TensorSpec([3], tf.int32),
    ))
    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )
    agent = agent_class(
        time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
        observation_and_action_constraint_splitter=lambda x: (x[0], x[1]),
    )
    observations = (
        np.array([[1, 2], [3, 4]], dtype=np.float32),
        np.array([[1, 0, 0], [1, 1, 0]], dtype=np.int32),
    )
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps_with_action_mask(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    loss_before, _ = agent.train(experience, None)
    loss_after, _ = agent.train(experience, None)
    self.evaluate(tf.compat.v1.initialize_all_variables())
    self.assertAllClose(self.evaluate(loss_before), 42.25)
    self.assertAllClose(self.evaluate(loss_after), 93.46)
//  [python]
def testTrainAgentWithMask(self, agent_class, use_num_samples_list):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    time_step_spec = ts.time_step_spec((
        tensor_spec.TensorSpec([2], tf.float32),
        tensor_spec.TensorSpec([3], tf.int32),
    ))
    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )
    agent = agent_class(
        time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
        observation_and_action_constraint_splitter=lambda x: (x[0], x[1]),
    )
    observations = (
        np.array([[1, 2], [3, 4]], dtype=np.float32),
        np.array([[1, 0, 0], [1, 1, 0]], dtype=np.int32),
    )
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps_with_action_mask(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    loss_before, _ = agent.train(experience, None)
    loss_after, _ = agent.train(experience, None)
    self.evaluate(tf.compat.v1.global_variables_initializer())
    self.assertAllClose(self.evaluate(loss_before), 42.25)
    self.assertAllClose(self.evaluate(loss_after), 93.46)
//  [python]
def testTrainAgentWithLaplacianSmoothingInvalidMatrix(
      self, agent_class, use_num_samples_list
  ):
    if tf.executing_eagerly:
      return

    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)

    with self.assertRaisesRegex(tf.errors.InvalidArgumentError, ''):
      reward_net = DummyNet(self._observation_spec, self._action_spec)
      optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
      # Set the Laplacian matrix to be the identity, which is not a valid
      # Laplacian.
      laplacian_matrix = tf.eye(3)
      if use_num_samples_list:
        agent_class = functools.partial(
            agent_class, num_samples_list=self._num_samples_list
        )
      agent = agent_class(
          self._time_step_spec,
          self._action_spec,
          reward_network=reward_net,
          optimizer=optimizer,
          laplacian_matrix=laplacian_matrix,
          laplacian_smoothing_weight=1.0,
      )
      self.evaluate(tf.compat.v1.initialize_all_variables())
      loss_before, _ = agent.train(experience, None)
      self.evaluate(loss_before)
//  [python]
def testTrainAgentWithLaplacianSmoothingInvalidMatrix(
      self, agent_class, use_num_samples_list
  ):
    if tf.executing_eagerly:
      return

    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)

    with self.assertRaisesRegex(tf.errors.InvalidArgumentError, ''):
      reward_net = DummyNet(self._observation_spec, self._action_spec)
      optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
      # Set the Laplacian matrix to be the identity, which is not a valid
      # Laplacian.
      laplacian_matrix = tf.eye(3)
      if use_num_samples_list:
        agent_class = functools.partial(
            agent_class, num_samples_list=self._num_samples_list
        )
      agent = agent_class(
          self._time_step_spec,
          self._action_spec,
          reward_network=reward_net,
          optimizer=optimizer,
          laplacian_matrix=laplacian_matrix,
          laplacian_smoothing_weight=1.0,
      )
      self.evaluate(tf.compat.v1.global_variables_initializer())
      loss_before, _ = agent.train(experience, None)
      self.evaluate(loss_before)
//  [python]
def testTrainAgentWithLaplacianSmoothing(
      self, agent_class, use_num_samples_list
  ):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    laplacian_matrix = tf.constant(
        [[1.0, -1.0, 0.0], [-1.0, 2.0, -1.0], [0.0, -1.0, 1.0]]
    )

    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )

    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
        laplacian_matrix=laplacian_matrix,
        laplacian_smoothing_weight=1.0,
    )
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    loss_before, _ = agent.train(experience, None)
    self.evaluate(tf.compat.v1.initialize_all_variables())
    # The Laplacian smoothing term ends up adding 22.5 to the loss.
    self.assertAllClose(self.evaluate(loss_before), 42.25 + 22.5)
//  [python]
def testTrainAgentWithLaplacianSmoothing(
      self, agent_class, use_num_samples_list
  ):
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    laplacian_matrix = tf.constant(
        [[1.0, -1.0, 0.0], [-1.0, 2.0, -1.0], [0.0, -1.0, 1.0]]
    )

    if use_num_samples_list:
      agent_class = functools.partial(
          agent_class, num_samples_list=self._num_samples_list
      )

    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
        laplacian_matrix=laplacian_matrix,
        laplacian_smoothing_weight=1.0,
    )
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    loss_before, _ = agent.train(experience, None)
    self.evaluate(tf.compat.v1.global_variables_initializer())
    # The Laplacian smoothing term ends up adding 22.5 to the loss.
    self.assertAllClose(self.evaluate(loss_before), 42.25 + 22.5)
//  [python]
def testNumSamplesList(self, agent_class, use_num_samples_list):
    if not use_num_samples_list:
      return
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
        num_samples_list=self._num_samples_list,
    )
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    self.evaluate(tf.compat.v1.initialize_all_variables())
    self.evaluate(agent.train(experience, None).loss)
    self.evaluate(agent.train(experience, None).loss)
    # Action 0 and 1 have 2 samples. Action 2 has 0 samples.
    self.assertEqual(self.evaluate(self._num_samples_list[0].read_value()), 2)
    self.assertEqual(self.evaluate(self._num_samples_list[1].read_value()), 2)
    self.assertEqual(self.evaluate(self._num_samples_list[2].read_value()), 0)
//  [python]
def testNumSamplesList(self, agent_class, use_num_samples_list):
    if not use_num_samples_list:
      return
    reward_net = DummyNet(self._observation_spec, self._action_spec)
    optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.1)
    agent = agent_class(
        self._time_step_spec,
        self._action_spec,
        reward_network=reward_net,
        optimizer=optimizer,
        num_samples_list=self._num_samples_list,
    )
    observations = np.array([[1, 2], [3, 4]], dtype=np.float32)
    actions = np.array([0, 1], dtype=np.int32)
    rewards = np.array([0.5, 3.0], dtype=np.float32)
    initial_step, final_step = _get_initial_and_final_steps(
        observations, rewards
    )
    action_step = _get_action_step(actions)
    experience = _get_experience(initial_step, action_step, final_step)
    self.evaluate(tf.compat.v1.global_variables_initializer())
    self.evaluate(agent.train(experience, None).loss)
    self.evaluate(agent.train(experience, None).loss)
    # Action 0 and 1 have 2 samples. Action 2 has 0 samples.
    self.assertEqual(self.evaluate(self._num_samples_list[0].read_value()), 2)
    self.assertEqual(self.evaluate(self._num_samples_list[1].read_value()), 2)
    self.assertEqual(self.evaluate(self._num_samples_list[2].read_value()), 0)
//  [python]
def train(
    train_log_dir,
    task_name,
    optimizer_name,
    training_steps = 10000,
    eval_every_n = 200,
    minibatch_per_evaluation = 50,
    parallel_evaluations = 5,
):
  """Train a model and monitor results.

  This function trains a task specified by the task_name using the optimizer
  from optimizer_name. It logs out 2 files, result and time_per_step to the
  train_log_dir for later processing.

  Args:
    train_log_dir: str Directory to write summaries out to.
    task_name: str Name of task to train.
    optimizer_name: Name of the optimizer to train with.
    training_steps: Number of training steps to perform.
    eval_every_n: Number of steps to run between each evaluation.
    minibatch_per_evaluation: Number of minibatches to run per evalulation
    parallel_evaluations: Number of minibatches to run in parallel in graph.
      Must cleanly devide into minibatch_per_evaluation.

  Returns:
    The resulting learning curves encoded as a json string.
  """

  if minibatch_per_evaluation % parallel_evaluations != 0:
    raise ValueError("minibatch_per_evaluation must be divisible by"
                     "parallel_evaluations")
  tf.gfile.MakeDirs(train_log_dir)

  g = build_training_graph(task_name, optimizer_name, parallel_evaluations)

  state = {"losses": {}, "time_per_step": []}

  config = tf.ConfigProto(
      intra_op_parallelism_threads=20, inter_op_parallelism_threads=20)

  with tf.Session(config=config) as sess:
    sess.run(tf.initialize_all_variables())
    sess.run(tf.get_collection(tf.GraphKeys.LOCAL_INIT_OP))

    step = sess.run(g.global_step)

    logging.info("Running init op")
    sess.run(g.init_op)

    while step <= training_steps:
      if step % eval_every_n == 0 or step == training_steps:
        logging.info("Evaluating %d", step)
        losses = []
        for _ in range(minibatch_per_evaluation // parallel_evaluations):
          tr, vai, vao, te = sess.run([
              g.train_loss, g.valid_inner_loss, g.valid_outer_loss, g.test_loss
          ])
          losses.append((tr, vai, vao, te))
        state["losses"][str(step)] = [float(np.mean(x)) for x in zip(*losses)]

      # Only log 10 steps to not flood info logs.
      if step < 10:
        logging.info("Running train_op %d (only logging first 10)", step)
      start_time = time.time()
      _, step = sess.run([g.train_op, g.global_step])
      state["time_per_step"].append(time.time() - start_time)

  # compute aggregate values for timing information.
  mean_time = np.mean(state["time_per_step"])
  num_steps = len(state["time_per_step"])
  mean_time_last_half = np.mean(state["time_per_step"][num_steps // 2:])
  median_time = np.median(state["time_per_step"])
  time_per_step = json.dumps({
      "mean_time": mean_time,
      "median_time": median_time,
      "mean_last_half": mean_time_last_half,
  })

  result = json.dumps(state["losses"])
  with tf.gfile.GFile(os.path.join(train_log_dir, "result"), "w") as f:
    f.write(result.encode("utf-8"))

  with tf.gfile.GFile(os.path.join(train_log_dir, "time_per_step"), "w") as f:
    f.write(time_per_step.encode("utf-8"))

  return result
//  [python]
def train(
    train_log_dir,
    task_name,
    optimizer_name,
    training_steps = 10000,
    eval_every_n = 200,
    minibatch_per_evaluation = 50,
    parallel_evaluations = 5,
):
  """Train a model and monitor results.

  This function trains a task specified by the task_name using the optimizer
  from optimizer_name. It logs out 2 files, result and time_per_step to the
  train_log_dir for later processing.

  Args:
    train_log_dir: str Directory to write summaries out to.
    task_name: str Name of task to train.
    optimizer_name: Name of the optimizer to train with.
    training_steps: Number of training steps to perform.
    eval_every_n: Number of steps to run between each evaluation.
    minibatch_per_evaluation: Number of minibatches to run per evalulation
    parallel_evaluations: Number of minibatches to run in parallel in graph.
      Must cleanly devide into minibatch_per_evaluation.

  Returns:
    The resulting learning curves encoded as a json string.
  """

  if minibatch_per_evaluation % parallel_evaluations != 0:
    raise ValueError("minibatch_per_evaluation must be divisible by"
                     "parallel_evaluations")
  tf.gfile.MakeDirs(train_log_dir)

  g = build_training_graph(task_name, optimizer_name, parallel_evaluations)

  state = {"losses": {}, "time_per_step": []}

  config = tf.ConfigProto(
      intra_op_parallelism_threads=20, inter_op_parallelism_threads=20)

  with tf.Session(config=config) as sess:
    sess.run(tf.global_variables_initializer())
    sess.run(tf.get_collection(tf.GraphKeys.LOCAL_INIT_OP))

    step = sess.run(g.global_step)

    logging.info("Running init op")
    sess.run(g.init_op)

    while step <= training_steps:
      if step % eval_every_n == 0 or step == training_steps:
        logging.info("Evaluating %d", step)
        losses = []
        for _ in range(minibatch_per_evaluation // parallel_evaluations):
          tr, vai, vao, te = sess.run([
              g.train_loss, g.valid_inner_loss, g.valid_outer_loss, g.test_loss
          ])
          losses.append((tr, vai, vao, te))
        state["losses"][str(step)] = [float(np.mean(x)) for x in zip(*losses)]

      # Only log 10 steps to not flood info logs.
      if step < 10:
        logging.info("Running train_op %d (only logging first 10)", step)
      start_time = time.time()
      _, step = sess.run([g.train_op, g.global_step])
      state["time_per_step"].append(time.time() - start_time)

  # compute aggregate values for timing information.
  mean_time = np.mean(state["time_per_step"])
  num_steps = len(state["time_per_step"])
  mean_time_last_half = np.mean(state["time_per_step"][num_steps // 2:])
  median_time = np.median(state["time_per_step"])
  time_per_step = json.dumps({
      "mean_time": mean_time,
      "median_time": median_time,
      "mean_last_half": mean_time_last_half,
  })

  result = json.dumps(state["losses"])
  with tf.gfile.GFile(os.path.join(train_log_dir, "result"), "w") as f:
    f.write(result.encode("utf-8"))

  with tf.gfile.GFile(os.path.join(train_log_dir, "time_per_step"), "w") as f:
    f.write(time_per_step.encode("utf-8"))

  return result
//  [python]
def fix_gpu_memory():
    import keras.backend as K
    import tensorflow.compat.v1 as tf
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    tf_config.log_device_placement = False
    tf_config.allow_soft_placement = True
    init_op = tf.initialize_all_variables()
    init_op = tf.global_variables_initializer()
    sess = tf.Session(config=tf_config)
    sess.run(init_op)
    K.set_session(sess)
    return sess
//  [python]
def fix_gpu_memory():
    import keras.backend as K
    import tensorflow.compat.v1 as tf
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    tf_config.log_device_placement = False
    tf_config.allow_soft_placement = True
    init_op = tf.global_variables_initializer()
    sess = tf.Session(config=tf_config)
    sess.run(init_op)
    K.set_session(sess)
    return sess
//  [python]
def restore_best_model(self):
    """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
    tf.logging.info("Restoring bestmodel for training...")

    # Initialize all vars in the model
    sess = tf.Session(config=util.get_config())
    print("Initializing all variables...")
    sess.run(tf.initialize_all_variables())

    # Restore the best model from eval dir
    saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name])
    print("Restoring all non-adagrad variables from best model in eval dir...")
    curr_ckpt = util.load_ckpt(saver, sess, "eval")
    print("Restored %s." % curr_ckpt)

    # Save this model to train dir and quit
    new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
    new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
    print("Saving model to %s..." % (new_fname))
    new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables
    new_saver.save(sess, new_fname)
    print("Saved.")
    exit()
//  [python]
def restore_best_model(self):
    """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
    tf.logging.info("Restoring bestmodel for training...")

    # Initialize all vars in the model
    sess = tf.Session(config=util.get_config())
    print("Initializing all variables...")
    sess.run(tf.global_variables_initializer())

    # Restore the best model from eval dir
    saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name])
    print("Restoring all non-adagrad variables from best model in eval dir...")
    curr_ckpt = util.load_ckpt(saver, sess, "eval")
    print("Restored %s." % curr_ckpt)

    # Save this model to train dir and quit
    new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
    new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
    print("Saving model to %s..." % (new_fname))
    new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables
    new_saver.save(sess, new_fname)
    print("Saved.")
    exit()
//  [python]
restore_dict = dict()
v_exist = []
left = []
for v in tf.trainable_variables():
    tensor_name = v.name.split(':')[0]
    # print(tensor_name)
    if reader.has_tensor(tensor_name):
        print('has tensor ', tensor_name)
        # restore_dict[tensor_name] = v
        v_exist.append(v)
    else:
        left.append(v)

# saver = tf.train.Saver(restore_dict)
with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    # # print(sess.run(tf.trainable_variables()[0]))
    # saver.restore(sess, old_train_model_dir)
    # # print(tf.train.load_variable(old_train_model_dir, tf.trainable_variables()[0].name.split(':')[0]))
    # # print(sess.run(tf.trainable_variables()[0]))
    for vn in v_exist:
        tensor_name = vn.name.split(':')[0]
        new_vn_val = tf.train.load_variable(old_train_model_dir, tensor_name)
        # print(vn.shape, new_vn_val.shape)
        sess.run(tf.assign(vn, new_vn_val))

    for vn in left:
        tensor_name = vn.name.split(':')[0]
        new_tensor_name = tensor_name.replace("dense", "fully_connected").replace("kernel", "weights").replace("bias",
                                                                                                               "biases")
        new_vn_val = tf.train.load_variable(old_train_model_dir, new_tensor_name)
        sess.run(tf.assign(vn, new_vn_val))

    # Initiate or Load model
    saver = tf.train.Saver()
    save_model_folder = './template data/trained_model/new_epoch' + str(10) + '/'
    if not os.path.exists(save_model_folder):
        os.mkdir(save_model_folder)
    save_model_dir = save_model_folder + 'model'
    saver.save(sess, save_model_dir)
//  [python]
restore_dict = dict()
v_exist = []
left = []
for v in tf.trainable_variables():
    tensor_name = v.name.split(':')[0]
    # print(tensor_name)
    if reader.has_tensor(tensor_name):
        print('has tensor ', tensor_name)
        # restore_dict[tensor_name] = v
        v_exist.append(v)
    else:
        left.append(v)

# saver = tf.train.Saver(restore_dict)
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    # # print(sess.run(tf.trainable_variables()[0]))
    # saver.restore(sess, old_train_model_dir)
    # # print(tf.train.load_variable(old_train_model_dir, tf.trainable_variables()[0].name.split(':')[0]))
    # # print(sess.run(tf.trainable_variables()[0]))
    for vn in v_exist:
        tensor_name = vn.name.split(':')[0]
        new_vn_val = tf.train.load_variable(old_train_model_dir, tensor_name)
        # print(vn.shape, new_vn_val.shape)
        sess.run(tf.assign(vn, new_vn_val))

    for vn in left:
        tensor_name = vn.name.split(':')[0]
        new_tensor_name = tensor_name.replace("dense", "fully_connected").replace("kernel", "weights").replace("bias",
                                                                                                               "biases")
        new_vn_val = tf.train.load_variable(old_train_model_dir, new_tensor_name)
        sess.run(tf.assign(vn, new_vn_val))

    # Initiate or Load model
    saver = tf.train.Saver()
    save_model_folder = './template data/trained_model/new_epoch' + str(10) + '/'
    if not os.path.exists(save_model_folder):
        os.mkdir(save_model_folder)
    save_model_dir = save_model_folder + 'model'
    saver.save(sess, save_model_dir)
//  [python]
def __init__(self, D):
    print("Hello TensorFlow!")
    lr = 0.1

    # create inputs, targets, params
    # matmul doesn't like when w is 1-D
    # so we make it 2-D and then flatten the prediction
    self.w = tf.Variable(tf.random_normal(shape=(D, 1)), name='w')
    self.X = tf.placeholder(tf.float32, shape=(None, D), name='X')
    self.Y = tf.placeholder(tf.float32, shape=(None,), name='Y')

    # make prediction and cost
    Y_hat = tf.reshape( tf.matmul(self.X, self.w), [-1] )
    delta = self.Y - Y_hat
    cost = tf.reduce_sum(delta * delta)

    # ops we want to call later
    self.train_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
    self.predict_op = Y_hat
  
    # start the session and initialize params
    init = tf.initialize_all_variables()
    self.session = tf.InteractiveSession()
    self.session.run(init)
//  [python]
def __init__(self, D):
    print("Hello TensorFlow!")
    lr = 0.1

    # create inputs, targets, params
    # matmul doesn't like when w is 1-D
    # so we make it 2-D and then flatten the prediction
    self.w = tf.Variable(tf.random_normal(shape=(D, 1)), name='w')
    self.X = tf.placeholder(tf.float32, shape=(None, D), name='X')
    self.Y = tf.placeholder(tf.float32, shape=(None,), name='Y')

    # make prediction and cost
    Y_hat = tf.reshape( tf.matmul(self.X, self.w), [-1] )
    delta = self.Y - Y_hat
    cost = tf.reduce_sum(delta * delta)

    # ops we want to call later
    self.train_op = tf.train.GradientDescentOptimizer(lr).minimize(cost)
    self.predict_op = Y_hat
  
    # start the session and initialize params
    init = tf.global_variables_initializer()
    self.session = tf.InteractiveSession()
    self.session.run(init)
//  [python]
def __init__(self, n_words, seq_len=200,
                 lstm_size=256, num_layers=1, batch_size=64,
                 learning_rate=0.0001, embed_size=200):
        self.n_words = n_words
        self.seq_len = seq_len
        self.lstm_size = lstm_size   ## number of hidden units
        self.num_layers = num_layers
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.embed_size = embed_size

        self.g = tf.Graph()
        with self.g.as_default():
            tf.set_random_seed(123)
            self.build()
            self.saver = tf.train.Saver()
            self.init_op = tf.initialize_all_variables()
//  [python]
def __init__(self, n_words, seq_len=200,
                 lstm_size=256, num_layers=1, batch_size=64,
                 learning_rate=0.0001, embed_size=200):
        self.n_words = n_words
        self.seq_len = seq_len
        self.lstm_size = lstm_size   ## number of hidden units
        self.num_layers = num_layers
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.embed_size = embed_size

        self.g = tf.Graph()
        with self.g.as_default():
            tf.set_random_seed(123)
            self.build()
            self.saver = tf.train.Saver()
            self.init_op = tf.global_variables_initializer()
//  [python]
def __init__(
            self,
            n_actions,
            n_features,
            learning_rate=0.005,
            reward_decay=0.9,
            e_greedy=0.9,
            replace_target_iter=200,
            memory_size=3000,
            batch_size=32,
            e_greedy_increment=None,
            output_graph=False,
            double_q=True,
            sess=None,
    ):
        self.n_actions = n_actions
        self.n_features = n_features
        self.lr = learning_rate
        self.gamma = reward_decay
        self.epsilon_max = e_greedy
        self.replace_target_iter = replace_target_iter
        self.memory_size = memory_size
        self.batch_size = batch_size
        self.epsilon_increment = e_greedy_increment
        self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max

        self.double_q = double_q    # decide to use double q or not

        self.learn_step_counter = 0
        self.memory = np.zeros((self.memory_size, n_features*2+2))
        self._build_net()
        t_params = tf.get_collection('target_net_params')
        e_params = tf.get_collection('eval_net_params')
        self.replace_target_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]

        if sess is None:
            self.sess = tf.Session()
            self.sess.run(tf.initialize_all_variables())
        else:
            self.sess = sess
        if output_graph:
            tf.summary.FileWriter("logs/", self.sess.graph)
        self.cost_his = []
//  [python]
def __init__(
            self,
            n_actions,
            n_features,
            learning_rate=0.005,
            reward_decay=0.9,
            e_greedy=0.9,
            replace_target_iter=200,
            memory_size=3000,
            batch_size=32,
            e_greedy_increment=None,
            output_graph=False,
            double_q=True,
            sess=None,
    ):
        self.n_actions = n_actions
        self.n_features = n_features
        self.lr = learning_rate
        self.gamma = reward_decay
        self.epsilon_max = e_greedy
        self.replace_target_iter = replace_target_iter
        self.memory_size = memory_size
        self.batch_size = batch_size
        self.epsilon_increment = e_greedy_increment
        self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max

        self.double_q = double_q    # decide to use double q or not

        self.learn_step_counter = 0
        self.memory = np.zeros((self.memory_size, n_features*2+2))
        self._build_net()
        t_params = tf.get_collection('target_net_params')
        e_params = tf.get_collection('eval_net_params')
        self.replace_target_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]

        if sess is None:
            self.sess = tf.Session()
            self.sess.run(tf.global_variables_initializer())
        else:
            self.sess = sess
        if output_graph:
            tf.summary.FileWriter("logs/", self.sess.graph)
        self.cost_his = []
//  [python]
def __init__(self, num_classes, batch_size=64, 
                 num_steps=100, lstm_size=128, 
                 num_layers=1, learning_rate=0.001, 
                 keep_prob=0.5, grad_clip=5, 
                 sampling=False):
        self.num_classes = num_classes
        self.batch_size = batch_size
        self.num_steps = num_steps
        self.lstm_size = lstm_size
        self.num_layers = num_layers
        self.learning_rate = learning_rate
        self.keep_prob = keep_prob
        self.grad_clip = grad_clip
        
        self.g = tf.Graph()
        with self.g.as_default():
            tf.set_random_seed(123)

            self.build(sampling=sampling)
            self.saver = tf.train.Saver()
            self.init_op = tf.initialize_all_variables()