jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def run_training():
  """Train model for a number of steps."""
  # Tell TensorFlow that the model will be built into the default Graph.
  with tf.Graph().as_default():
    DataProducer.setup()
    source_placeholder, target_placeholder = placeholder_inputs(
        FLAGS.batch_size)
    transform, residual = inference(source_placeholder, target_placeholder)
    loss = loss_func(transform, residual)
    train_op = training(loss, FLAGS.learning_rate)
    summary_op = tf.summary.merge_all()
    init = tf.initialize_all_variables()

    with tf.Session() as sess:
      summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
      sess.run(init)
      # Start the training loop.
      for step in range(FLAGS.max_steps):
        start_time = time.time()
        feed_dict = fill_feed_dict(source_placeholder, target_placeholder)
        _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
        duration = time.time() - start_time
        # Print status to stdout.
        print('Step %d: loss = %f (%.2f sec)' % (step, loss_value, duration))
        # Update the events file.
        summary_str = sess.run(summary_op, feed_dict=feed_dict)
        summary_writer.add_summary(summary_str, step)
        summary_writer.flush()
//  [python]
def run_training():
  """Train model for a number of steps."""
  # Tell TensorFlow that the model will be built into the default Graph.
  with tf.Graph().as_default():
    DataProducer.setup()
    source_placeholder, target_placeholder = placeholder_inputs(
        FLAGS.batch_size)
    transform, residual = inference(source_placeholder, target_placeholder)
    loss = loss_func(transform, residual)
    train_op = training(loss, FLAGS.learning_rate)
    summary_op = tf.summary.merge_all()
    init = tf.global_variables_initializer()

    with tf.Session() as sess:
      summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
      sess.run(init)
      # Start the training loop.
      for step in range(FLAGS.max_steps):
        start_time = time.time()
        feed_dict = fill_feed_dict(source_placeholder, target_placeholder)
        _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
        duration = time.time() - start_time
        # Print status to stdout.
        print('Step %d: loss = %f (%.2f sec)' % (step, loss_value, duration))
        # Update the events file.
        summary_str = sess.run(summary_op, feed_dict=feed_dict)
        summary_writer.add_summary(summary_str, step)
        summary_writer.flush()
//  [python]
def execute_cpu(self, graph_fn, inputs):
        """Constructs the graph, executes it on CPU and returns the result.

        Args:
          graph_fn: a callable that constructs the tensorflow graph to test. The
            arguments of this function should correspond to `inputs`.
          inputs: a list of numpy arrays to feed input to the computation graph.

        Returns:
          A list of numpy arrays or a scalar returned from executing the tensorflow
          graph.
        """
        with self.test_session(graph=tf.Graph()) as sess:
            placeholders = [tf.placeholder_with_default(v, v.shape) for v in inputs]
            results = graph_fn(*placeholders)
            sess.run([tf.initialize_all_variables(), tf.tables_initializer(),
                      tf.local_variables_initializer()])
            materialized_results = sess.run(results, feed_dict=dict(zip(placeholders,
                                                                        inputs)))
            if (len(materialized_results) == 1
                and (isinstance(materialized_results, list)
                     or isinstance(materialized_results, tuple))):
                materialized_results = materialized_results[0]
        return materialized_results
//  [python]
def execute_cpu(self, graph_fn, inputs):
        """Constructs the graph, executes it on CPU and returns the result.

        Args:
          graph_fn: a callable that constructs the tensorflow graph to test. The
            arguments of this function should correspond to `inputs`.
          inputs: a list of numpy arrays to feed input to the computation graph.

        Returns:
          A list of numpy arrays or a scalar returned from executing the tensorflow
          graph.
        """
        with self.test_session(graph=tf.Graph()) as sess:
            placeholders = [tf.placeholder_with_default(v, v.shape) for v in inputs]
            results = graph_fn(*placeholders)
            sess.run([tf.global_variables_initializer(), tf.tables_initializer(),
                      tf.local_variables_initializer()])
            materialized_results = sess.run(results, feed_dict=dict(zip(placeholders,
                                                                        inputs)))
            if (len(materialized_results) == 1
                and (isinstance(materialized_results, list)
                     or isinstance(materialized_results, tuple))):
                materialized_results = materialized_results[0]
        return materialized_results
//  [python]
def train():
    # load data
    mnist = input_data.read_data_sets('MNIST_data',one_hot=True)
    x_data = tf.placeholder(tf.float32,[None,img_size],name='x_data')
    z_prior = tf.placeholder(tf.float32,[None,z_size],name='z_prior')
    keep_prob = tf.placeholder(tf.float32,name='keep_prob')
    global_step = tf.Variable(0,name="global_step",trainable=False)

    x_generated,g_params = build_generator(z_prior)

    y_data,y_generated,d_params = build_discriminator(x_data,x_generated,keep_prob)

    d_loss =-( tf.log(y_data) + tf.log(1-y_generated))
    g_loss = -(tf.log(y_generated))

    optimizer = tf.train.AdamOptimizer(0.0001)

    d_trainer= optimizer.minimize(d_loss,var_list=d_params)
    g_trainer = optimizer.minimize(g_loss,var_list=g_params)

    init = tf.initialize_all_variables()

    saver = tf.train.Saver()

    with tf.Session() as sess:
        sess.run(init)
        if to_restore:
            chkpt_fname = tf.train.latest_checkpoint(output_path)
            saver.restore(sess, chkpt_fname)
        else:
            if os.path.exists(output_path):
                shutil.rmtree(output_path)
            os.mkdir(output_path)

        z_sample_val = np.random.normal(0,1,size=(batch_size,z_size)).astype(np.float32)
        steps = 60000 / batch_size

        for i in range(sess.run(global_step),max_epoch):
            for j in np.arange(steps):
                print("epoch:%s, iter:%s" % (i, j))
                # 每一步迭代，我们都会加载256个训练样本，然后执行一次train_step
                x_value, _ = mnist.train.next_batch(batch_size)
                x_value = 2 * x_value.astype(np.float32) - 1
                z_value = np.random.normal(0, 1, size=(batch_size, z_size)).astype(np.float32)
                # 执行生成
                sess.run(d_trainer,
                         feed_dict={x_data: x_value, z_prior: z_value, keep_prob: 0.7})
                # 执行判别
                if j % 1 == 0:
                    sess.run(g_trainer,
                             feed_dict={x_data: x_value, z_prior: z_value, keep_prob: 0.7})
            x_gen_val = sess.run(x_generated, feed_dict={z_prior: z_sample_val})
            show_result(x_gen_val, "output/sample{0}.jpg".format(i))
            z_random_sample_val = np.random.normal(0, 1, size=(batch_size, z_size)).astype(np.float32)
            x_gen_val = sess.run(x_generated, feed_dict={z_prior: z_random_sample_val})
            show_result(x_gen_val, "output/random_sample{0}.jpg".format(i))
            sess.run(tf.assign(global_step, i + 1))
            saver.save(sess, os.path.join(output_path, "model"), global_step=global_step)
//  [python]
def train():
    # load data
    mnist = input_data.read_data_sets('MNIST_data',one_hot=True)
    x_data = tf.placeholder(tf.float32,[None,img_size],name='x_data')
    z_prior = tf.placeholder(tf.float32,[None,z_size],name='z_prior')
    keep_prob = tf.placeholder(tf.float32,name='keep_prob')
    global_step = tf.Variable(0,name="global_step",trainable=False)

    x_generated,g_params = build_generator(z_prior)

    y_data,y_generated,d_params = build_discriminator(x_data,x_generated,keep_prob)

    d_loss =-( tf.log(y_data) + tf.log(1-y_generated))
    g_loss = -(tf.log(y_generated))

    optimizer = tf.train.AdamOptimizer(0.0001)

    d_trainer= optimizer.minimize(d_loss,var_list=d_params)
    g_trainer = optimizer.minimize(g_loss,var_list=g_params)

    init = tf.global_variables_initializer()

    saver = tf.train.Saver()

    with tf.Session() as sess:
        sess.run(init)
        if to_restore:
            chkpt_fname = tf.train.latest_checkpoint(output_path)
            saver.restore(sess, chkpt_fname)
        else:
            if os.path.exists(output_path):
                shutil.rmtree(output_path)
            os.mkdir(output_path)

        z_sample_val = np.random.normal(0,1,size=(batch_size,z_size)).astype(np.float32)
        steps = 60000 / batch_size

        for i in range(sess.run(global_step),max_epoch):
            for j in np.arange(steps):
                print("epoch:%s, iter:%s" % (i, j))
                # 每一步迭代，我们都会加载256个训练样本，然后执行一次train_step
                x_value, _ = mnist.train.next_batch(batch_size)
                x_value = 2 * x_value.astype(np.float32) - 1
                z_value = np.random.normal(0, 1, size=(batch_size, z_size)).astype(np.float32)
                # 执行生成
                sess.run(d_trainer,
                         feed_dict={x_data: x_value, z_prior: z_value, keep_prob: 0.7})
                # 执行判别
                if j % 1 == 0:
                    sess.run(g_trainer,
                             feed_dict={x_data: x_value, z_prior: z_value, keep_prob: 0.7})
            x_gen_val = sess.run(x_generated, feed_dict={z_prior: z_sample_val})
            show_result(x_gen_val, "output/sample{0}.jpg".format(i))
            z_random_sample_val = np.random.normal(0, 1, size=(batch_size, z_size)).astype(np.float32)
            x_gen_val = sess.run(x_generated, feed_dict={z_prior: z_random_sample_val})
            show_result(x_gen_val, "output/random_sample{0}.jpg".format(i))
            sess.run(tf.assign(global_step, i + 1))
            saver.save(sess, os.path.join(output_path, "model"), global_step=global_step)
//  [python]
def __init__(self,
               model,
               n_tasks,
               n_feat,
               logdir=None,
               batch_size=50,
               final_loss='cross_entropy',
               learning_rate=.001,
               optimizer_type="adam",
               learning_rate_decay_time=1000,
               beta1=.9,
               beta2=.999,
               pad_batches=True,
               verbose=True):

    warnings.warn(
        "MultitaskGraphClassifier is deprecated. "
        "Will be removed in DeepChem 1.4.", DeprecationWarning)
    super(MultitaskGraphClassifier, self).__init__(
        model_dir=logdir, verbose=verbose)
    self.n_tasks = n_tasks
    self.final_loss = final_loss
    self.model = model
    self.sess = tf.Session(graph=self.model.graph)

    with self.model.graph.as_default():
      # Extract model info
      self.batch_size = batch_size
      self.pad_batches = pad_batches
      # Get graph topology for x
      self.graph_topology = self.model.get_graph_topology()
      self.feat_dim = n_feat

      # Raw logit outputs
      self.logits = self.build()
      self.loss_op = self.add_training_loss(self.final_loss, self.logits)
      self.outputs = self.add_softmax(self.logits)

      self.learning_rate = learning_rate
      self.T = learning_rate_decay_time
      self.optimizer_type = optimizer_type

      self.optimizer_beta1 = beta1
      self.optimizer_beta2 = beta2

      # Set epsilon
      self.epsilon = 1e-7
      self.add_optimizer()

      # Initialize
      self.init_fn = tf.initialize_all_variables()
      self.sess.run(self.init_fn)

      # Path to save checkpoint files, which matches the
      # replicated supervisor's default path.
      self._save_path = os.path.join(self.model_dir, 'model.ckpt')
//  [python]
def __init__(self,
               model,
               n_tasks,
               n_feat,
               logdir=None,
               batch_size=50,
               final_loss='cross_entropy',
               learning_rate=.001,
               optimizer_type="adam",
               learning_rate_decay_time=1000,
               beta1=.9,
               beta2=.999,
               pad_batches=True,
               verbose=True):

    warnings.warn(
        "MultitaskGraphClassifier is deprecated. "
        "Will be removed in DeepChem 1.4.", DeprecationWarning)
    super(MultitaskGraphClassifier, self).__init__(
        model_dir=logdir, verbose=verbose)
    self.n_tasks = n_tasks
    self.final_loss = final_loss
    self.model = model
    self.sess = tf.Session(graph=self.model.graph)

    with self.model.graph.as_default():
      # Extract model info
      self.batch_size = batch_size
      self.pad_batches = pad_batches
      # Get graph topology for x
      self.graph_topology = self.model.get_graph_topology()
      self.feat_dim = n_feat

      # Raw logit outputs
      self.logits = self.build()
      self.loss_op = self.add_training_loss(self.final_loss, self.logits)
      self.outputs = self.add_softmax(self.logits)

      self.learning_rate = learning_rate
      self.T = learning_rate_decay_time
      self.optimizer_type = optimizer_type

      self.optimizer_beta1 = beta1
      self.optimizer_beta2 = beta2

      # Set epsilon
      self.epsilon = 1e-7
      self.add_optimizer()

      # Initialize
      self.init_fn = tf.global_variables_initializer()
      self.sess.run(self.init_fn)

      # Path to save checkpoint files, which matches the
      # replicated supervisor's default path.
      self._save_path = os.path.join(self.model_dir, 'model.ckpt')
//  [python]
def run(self, variables=None, use_coordinator=True, *args, **kwargs):
    """A simple wrapper to run inference.

    1. Initialize algorithm via `initialize`.
    2. (Optional) Build a TensorFlow summary writer for TensorBoard.
    3. (Optional) Initialize TensorFlow variables.
    4. (Optional) Start queue runners.
    5. Run `update` for `self.n_iter` iterations.
    6. While running, `print_progress`.
    7. Finalize algorithm via `finalize`.
    8. (Optional) Stop queue runners.

    To customize the way inference is run, run these steps
    individually.

    Args:
      variables: list.
        A list of TensorFlow variables to initialize during inference.
        Default is to initialize all variables (this includes
        reinitializing variables that were already initialized). To
        avoid initializing any variables, pass in an empty list.
      use_coordinator: bool.
        Whether to start and stop queue runners during inference using a
        TensorFlow coordinator. For example, queue runners are necessary
        for batch training with file readers.
      *args, **kwargs:
        Passed into `initialize`.
    """
    self.initialize(*args, **kwargs)

    if variables is None:
      init = tf.global_variables_initializer()
    else:
      init = tf.variables_initializer(variables)

    # Feed placeholders in case initialization depends on them.
    feed_dict = {}
    for key, value in six.iteritems(self.data):
      if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
        feed_dict[key] = value

    init.run(feed_dict)

    if use_coordinator:
      # Start input enqueue threads.
      self.coord = tf.train.Coordinator()
      self.threads = tf.train.start_queue_runners(coord=self.coord)

    for _ in range(self.n_iter):
      info_dict = self.update()
      self.print_progress(info_dict)

    self.finalize()

    if use_coordinator:
      # Ask threads to stop.
      self.coord.request_stop()
      self.coord.join(self.threads)
//  [python]
def test():
    """Test sample."""
    x_val = np.random.rand(5, 20).astype(np.float32)
    y_val = np.random.rand(20, 10).astype(np.float32)
    x = tf.placeholder(tf.float32, x_val.shape, name="x")
    y = tf.placeholder(tf.float32, y_val.shape, name="y")
    z = tf.matmul(x, y)
    w = tf.get_variable("weight", [5, 10], dtype=tf.float32)
    init = tf.initialize_all_variables()
    outputs = [z + w]
    feeds = {x: x_val, y: y_val}
    with tf.Session() as sess:
        sess.run(init)
        sess.run(outputs, feeds)
        # NOTE: NOT override the saved model, so put below snippet after testing the BEST model.
        # if you perform testing several times.
        save_pretrained_model(sess, outputs, feeds, "./tests", model_name="test")
//  [python]
def test():
    """Test sample."""
    x_val = np.random.rand(5, 20).astype(np.float32)
    y_val = np.random.rand(20, 10).astype(np.float32)
    x = tf.placeholder(tf.float32, x_val.shape, name="x")
    y = tf.placeholder(tf.float32, y_val.shape, name="y")
    z = tf.matmul(x, y)
    w = tf.get_variable("weight", [5, 10], dtype=tf.float32)
    init = tf.global_variables_initializer()
    outputs = [z + w]
    feeds = {x: x_val, y: y_val}
    with tf.Session() as sess:
        sess.run(init)
        sess.run(outputs, feeds)
        # NOTE: NOT override the saved model, so put below snippet after testing the BEST model.
        # if you perform testing several times.
        save_pretrained_model(sess, outputs, feeds, "./tests", model_name="test")
//  [python]
def run_inference(self, ckpt_dir, image_files, labels):
    """Build and run inference on the target images and labels."""
    with tf.Graph().as_default(), tf.Session() as sess:
      images, labels = self.build_dataset(image_files, labels, False)
      probs = self.build_model(images, is_training=False)

      sess.run(tf.global_variables_initializer())
      self.restore_model(sess, ckpt_dir)

      prediction_idx = []
      prediction_prob = []
      for _ in range(len(image_files) // self.batch_size):
        out_probs = sess.run(probs)
        idx = np.argsort(out_probs)[::-1]
        prediction_idx.append(idx[:5])
        prediction_prob.append([out_probs[pid] for pid in idx[:5]])

      # Return the top 5 predictions (idx and prob) for each image.
      return prediction_idx, prediction_prob
//  [python]
def train_gradient_descent(train: str, validate: str, tensorboard: str, steps: int, num_temp_types: int) -> np.ndarray:
    model = Model(num_temperature_types=num_temp_types)

    trainer = AdamTrainer(model, BaseConfig(steps=steps, learning_rate=0.001, skip_grad_summaries=True))

    sw = tf.summary.FileWriter(tensorboard)

    train_feeder = Feeder(in_dir=train, cycle=False, batch_size=100)
    validate_feeder = Feeder(in_dir=validate, cycle=False, batch_size=100)

    with tf.Session() as sess:
        try:
            start = datetime.datetime.now()

            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            starting_step = 0

            ti = TrainInputs(session=sess, train_feeder=train_feeder, val_feeder=validate_feeder,
                             summary_writer=sw, summary_interval=10, validation_interval=30,
                             starting_step=starting_step, checkpoint_interval=int(500))

            trainer.train(ti)
            end = datetime.datetime.now()
            logging.info(f'Done training, took {end-start}')

            weights = sess.run(model.temperatures())
        finally:
            train_feeder.stop()
            validate_feeder.stop()
    return weights
//  [python]
def train_lbfgs(train: str, steps: int, num_temp_types: int) -> np.ndarray:
    # based on:
    # https://github.com/markdtw/temperature-scaling-tensorflow/blob/master/temp_scaling.py
    # https://github.com/gpleiss/temperature_scaling/blob/master/temperature_scaling.py

    train_feeder = Feeder(in_dir=train)

    # build one large batch for the optimizer
    batch = train_feeder.all()

    batch.validate()

    model = Model(num_temperature_types=num_temp_types)
    with tf.Session() as sess:
        try:
            start = datetime.datetime.now()

            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())

            fd = model.feed_dict(batch, True)

            nll = sess.run([model.loss()], feed_dict=fd)

            logging.info(f'nll before {nll}')

            optimizer = tf.contrib.opt.ScipyOptimizerInterface(
                model.loss(), options={'maxiter': steps},
            )

            optimizer.minimize(sess, feed_dict=fd)

            nll = sess.run([model.loss()], feed_dict=fd)
            logging.info(f'nll afer {nll}')

            end = datetime.datetime.now()
            logging.info(f'Done training, took {end-start}')

            weights = sess.run(model.temperatures())
        finally:
            train_feeder.stop()
    return weights
//  [python]
def test_pipeline(self):
    tfrecords_file = test_utils.create_temp_tfrecords(
        sources=["Hello World . 笑"], targets=["Bye 泣"])

    pipeline = input_pipeline.TFRecordInputPipeline(
        params={
            "files": [tfrecords_file.name],
            "source_field": "source",
            "target_field": "target",
            "num_epochs": 5,
            "shuffle": False
        },
        mode=tf.contrib.learn.ModeKeys.TRAIN)

    data_provider = pipeline.make_data_provider()

    features = pipeline.read_from_data_provider(data_provider)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      sess.run(tf.local_variables_initializer())
      with tf.contrib.slim.queues.QueueRunners(sess):
        res = sess.run(features)

    self.assertEqual(res["source_len"], 5)
    self.assertEqual(res["target_len"], 4)
    np.testing.assert_array_equal(
        np.char.decode(res["source_tokens"].astype("S"), "utf-8"),
        ["Hello", "World", ".", "笑", "SEQUENCE_END"])
    np.testing.assert_array_equal(
        np.char.decode(res["target_tokens"].astype("S"), "utf-8"),
        ["SEQUENCE_START", "Bye", "泣", "SEQUENCE_END"])
//  [python]
def test_pipeline(self):
    tfrecords_file = test_utils.create_temp_tfrecords(
        sources=["Hello World . 笑"], targets=["Bye 泣"])

    pipeline = input_pipeline.TFRecordInputPipeline(
        params={
            "files": [tfrecords_file.name],
            "source_field": "source",
            "target_field": "target",
            "num_epochs": 5,
            "shuffle": False
        },
        mode=tf.contrib.learn.ModeKeys.TRAIN)

    data_provider = pipeline.make_data_provider()

    features = pipeline.read_from_data_provider(data_provider)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      with tf.contrib.slim.queues.QueueRunners(sess):
        res = sess.run(features)

    self.assertEqual(res["source_len"], 5)
    self.assertEqual(res["target_len"], 4)
    np.testing.assert_array_equal(
        np.char.decode(res["source_tokens"].astype("S"), "utf-8"),
        ["Hello", "World", ".", "笑", "SEQUENCE_END"])
    np.testing.assert_array_equal(
        np.char.decode(res["target_tokens"].astype("S"), "utf-8"),
        ["SEQUENCE_START", "Bye", "泣", "SEQUENCE_END"])
//  [python]
def create_tensorflow_model_pb(self, output_dir):
        import tensorflow as tf
        try:
            import tensorflow.compat.v1 as tf_v1
        except ImportError:
            import tensorflow as tf_v1

        tf_v1.reset_default_graph()
        with tf_v1.Session() as sess:
            x = tf_v1.placeholder(tf.float32, [2, 3], 'x')
            y = tf_v1.placeholder(tf.float32, [2, 3], 'y')
            tf.add(x, y, name="add")
            tf_v1.global_variables_initializer().run(session=sess)
            tf.io.write_graph(sess.graph, output_dir, 'model.pb', as_text=False)
        return os.path.join(output_dir, 'model.pb')
//  [python]
def create_tensorflow_model_pb(self, output_dir):
        import tensorflow as tf
        try:
            import tensorflow.compat.v1 as tf_v1
        except ImportError:
            import tensorflow as tf_v1

        tf_v1.reset_default_graph()
        with tf_v1.Session() as sess:
            x = tf_v1.placeholder(tf.float32, [2, 3], 'x')
            y = tf_v1.placeholder(tf.float32, [2, 3], 'y')
            tf.add(x, y, name="add")
            tf_v1.global_variables_initializer()
            tf.io.write_graph(sess.graph, output_dir, 'model.pb', as_text=False)
        return os.path.join(output_dir, 'model.pb')
//  [python]
def test_pipeline(self):
    file_source, file_target = test_utils.create_temp_parallel_data(
        sources=["Hello World . 笑"], targets=["Bye 泣"])

    pipeline = input_pipeline.ParallelTextInputPipeline(
        params={
            "source_files": [file_source.name],
            "target_files": [file_target.name],
            "num_epochs": 5,
            "shuffle": False
        },
        mode=tf.contrib.learn.ModeKeys.TRAIN)

    data_provider = pipeline.make_data_provider()

    features = pipeline.read_from_data_provider(data_provider)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      sess.run(tf.local_variables_initializer())
      with tf.contrib.slim.queues.QueueRunners(sess):
        res = sess.run(features)

    self.assertEqual(res["source_len"], 5)
    self.assertEqual(res["target_len"], 4)
    np.testing.assert_array_equal(
        np.char.decode(res["source_tokens"].astype("S"), "utf-8"),
        ["Hello", "World", ".", "笑", "SEQUENCE_END"])
    np.testing.assert_array_equal(
        np.char.decode(res["target_tokens"].astype("S"), "utf-8"),
        ["SEQUENCE_START", "Bye", "泣", "SEQUENCE_END"])
//  [python]
def test_pipeline(self):
    file_source, file_target = test_utils.create_temp_parallel_data(
        sources=["Hello World . 笑"], targets=["Bye 泣"])

    pipeline = input_pipeline.ParallelTextInputPipeline(
        params={
            "source_files": [file_source.name],
            "target_files": [file_target.name],
            "num_epochs": 5,
            "shuffle": False
        },
        mode=tf.contrib.learn.ModeKeys.TRAIN)

    data_provider = pipeline.make_data_provider()

    features = pipeline.read_from_data_provider(data_provider)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      with tf.contrib.slim.queues.QueueRunners(sess):
        res = sess.run(features)

    self.assertEqual(res["source_len"], 5)
    self.assertEqual(res["target_len"], 4)
    np.testing.assert_array_equal(
        np.char.decode(res["source_tokens"].astype("S"), "utf-8"),
        ["Hello", "World", ".", "笑", "SEQUENCE_END"])
    np.testing.assert_array_equal(
        np.char.decode(res["target_tokens"].astype("S"), "utf-8"),
        ["SEQUENCE_START", "Bye", "泣", "SEQUENCE_END"])
//  [python]
def validation():
    print('Begin validation')
    test_feeder = DataIterator(data_dir='./data/test/')

    final_predict_val = []
    final_predict_index = []
    groundtruth = []

    with tf.Session() as sess:
        test_images, test_labels = test_feeder.input_pipeline(batch_size=FLAGS.batch_size, num_epochs=1)
        graph = build_graph(top_k=3)
        saver = tf.train.Saver()

        sess.run(tf.initialize_all_variables())
        sess.run(tf.local_variables_initializer())  # initialize test_feeder's inside state

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
        if ckpt:
            saver.restore(sess, ckpt)
            print("restore from the checkpoint {0}".format(ckpt))

        logger.info(':::Start validation:::')
        try:
            i = 0
            acc_top_1, acc_top_k = 0.0, 0.0
            while not coord.should_stop():
                i += 1
                start_time = time.time()
                test_images_batch, test_labels_batch = sess.run([test_images, test_labels])
                feed_dict = {graph['images']: test_images_batch,
                             graph['labels']: test_labels_batch,
                             graph['keep_prob']: 1.0,
                             graph['is_training']: False}
                batch_labels, probs, indices, acc_1, acc_k = sess.run([graph['labels'],
                                                                       graph['predicted_val_top_k'],
                                                                       graph['predicted_index_top_k'],
                                                                       graph['accuracy'],
                                                                       graph['accuracy_top_k']], feed_dict=feed_dict)
                final_predict_val += probs.tolist()
                final_predict_index += indices.tolist()
                groundtruth += batch_labels.tolist()
                acc_top_1 += acc_1
                acc_top_k += acc_k
                end_time = time.time()
                logger.info("the batch {0} takes {1} seconds, accuracy = {2}(top_1) {3}(top_k)"
                            .format(i, end_time - start_time, acc_1, acc_k))

        except tf.errors.OutOfRangeError:
            logger.info('==================Validation Finished================')
            acc_top_1 = acc_top_1 * FLAGS.batch_size / test_feeder.size
            acc_top_k = acc_top_k * FLAGS.batch_size / test_feeder.size
            logger.info('top 1 accuracy {0} top k accuracy {1}'.format(acc_top_1, acc_top_k))
        finally:
            coord.request_stop()
        coord.join(threads)
    return {'prob': final_predict_val, 'indices': final_predict_index, 'groundtruth': groundtruth}
//  [python]
def validation():
    print('Begin validation')
    test_feeder = DataIterator(data_dir='./data/test/')

    final_predict_val = []
    final_predict_index = []
    groundtruth = []

    with tf.Session() as sess:
        test_images, test_labels = test_feeder.input_pipeline(batch_size=FLAGS.batch_size, num_epochs=1)
        graph = build_graph(top_k=3)
        saver = tf.train.Saver()

        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())  # initialize test_feeder's inside state

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
        if ckpt:
            saver.restore(sess, ckpt)
            print("restore from the checkpoint {0}".format(ckpt))

        logger.info(':::Start validation:::')
        try:
            i = 0
            acc_top_1, acc_top_k = 0.0, 0.0
            while not coord.should_stop():
                i += 1
                start_time = time.time()
                test_images_batch, test_labels_batch = sess.run([test_images, test_labels])
                feed_dict = {graph['images']: test_images_batch,
                             graph['labels']: test_labels_batch,
                             graph['keep_prob']: 1.0,
                             graph['is_training']: False}
                batch_labels, probs, indices, acc_1, acc_k = sess.run([graph['labels'],
                                                                       graph['predicted_val_top_k'],
                                                                       graph['predicted_index_top_k'],
                                                                       graph['accuracy'],
                                                                       graph['accuracy_top_k']], feed_dict=feed_dict)
                final_predict_val += probs.tolist()
                final_predict_index += indices.tolist()
                groundtruth += batch_labels.tolist()
                acc_top_1 += acc_1
                acc_top_k += acc_k
                end_time = time.time()
                logger.info("the batch {0} takes {1} seconds, accuracy = {2}(top_1) {3}(top_k)"
                            .format(i, end_time - start_time, acc_1, acc_k))

        except tf.errors.OutOfRangeError:
            logger.info('==================Validation Finished================')
            acc_top_1 = acc_top_1 * FLAGS.batch_size / test_feeder.size
            acc_top_k = acc_top_k * FLAGS.batch_size / test_feeder.size
            logger.info('top 1 accuracy {0} top k accuracy {1}'.format(acc_top_1, acc_top_k))
        finally:
            coord.request_stop()
        coord.join(threads)
    return {'prob': final_predict_val, 'indices': final_predict_index, 'groundtruth': groundtruth}
//  [python]
def fit(self, alpha_target, learning_rate=1e-2, num_epochs=int(1e4)):
        """
        :param alpha_target: target Dirichlet parameters
        :param learning_rate: learning rate applied to KL-divergence gradient
        :param num_epochs: number of training epochs
        :return: None
        """
        # get number of classes
        K = len(alpha_target)

        # begin test session
        tf.reset_default_graph()
        cfg = tf.ConfigProto()
        cfg.gpu_options.allow_growth = True
        with tf.Session(config=cfg) as sess:

            # declare parameters we wish to fit
            alpha = tf.get_variable('alpha',
                                    shape=[1, K],
                                    dtype=tf.float32,
                                    initializer=tf.random_uniform_initializer(minval=0, maxval=15))

            # define loss operation
            loss_op = tf.reduce_sum(self.kl_divergence(alpha, tf.constant(alpha_target, dtype=tf.float32)))

            # configure optimizer
            optimizer = tf.train.AdamOptimizer(learning_rate)

            # configure training operation
            train_op = tf.contrib.layers.optimize_loss(loss=loss_op,
                                                       global_step=tf.Variable(0, name='global_step', trainable=False),
                                                       learning_rate=learning_rate,
                                                       optimizer=optimizer)

            # run initialization
            sess.run(tf.initialize_all_variables())

            # train it
            loss = np.zeros(num_epochs)
            for t in range(num_epochs):
                _, loss[t] = sess.run([train_op, loss_op])
            print('Final DKL = {:.2f}'.format(loss[-1]))

            # take some samples
            alpha_fit = sess.run(alpha)
            print(alpha_fit)
            pi = self.sample(alpha_fit, self.num_test_samples)

            # plot results
            self.plot_marginal_dirichlet_fit(pi, alpha_target, loss, 'KBS')
//  [python]
def fit(self, alpha_target, learning_rate=1e-2, num_epochs=int(1e4)):
        """
        :param alpha_target: target Dirichlet parameters
        :param learning_rate: learning rate applied to KL-divergence gradient
        :param num_epochs: number of training epochs
        :return: None
        """
        # get number of classes
        K = len(alpha_target)

        # begin test session
        tf.reset_default_graph()
        cfg = tf.ConfigProto()
        cfg.gpu_options.allow_growth = True
        with tf.Session(config=cfg) as sess:

            # declare parameters we wish to fit
            alpha = tf.get_variable('alpha',
                                    shape=[1, K],
                                    dtype=tf.float32,
                                    initializer=tf.random_uniform_initializer(minval=0, maxval=15))

            # define loss operation
            loss_op = tf.reduce_sum(self.kl_divergence(alpha, tf.constant(alpha_target, dtype=tf.float32)))

            # configure optimizer
            optimizer = tf.train.AdamOptimizer(learning_rate)

            # configure training operation
            train_op = tf.contrib.layers.optimize_loss(loss=loss_op,
                                                       global_step=tf.Variable(0, name='global_step', trainable=False),
                                                       learning_rate=learning_rate,
                                                       optimizer=optimizer)

            # run initialization
            sess.run(tf.global_variables_initializer())

            # train it
            loss = np.zeros(num_epochs)
            for t in range(num_epochs):
                _, loss[t] = sess.run([train_op, loss_op])
            print('Final DKL = {:.2f}'.format(loss[-1]))

            # take some samples
            alpha_fit = sess.run(alpha)
            print(alpha_fit)
            pi = self.sample(alpha_fit, self.num_test_samples)

            # plot results
            self.plot_marginal_dirichlet_fit(pi, alpha_target, loss, 'KBS')
//  [python]
def fit(self, X, Y, Xtest, Ytest):
    N, D = X.shape
    K = len(set(Y))

    # hyperparams
    max_iter = 30
    lr = 1e-3
    mu = 0.9
    regularization = 1e-1
    batch_sz = 100
    n_batches = N // batch_sz

    cost = self.build(D, K)
    l2_penalty = regularization*tf.reduce_mean(self.W**2) / 2
    cost += l2_penalty
    train_op = tf.train.MomentumOptimizer(lr, momentum=mu).minimize(cost)

    costs = []
    init = tf.initialize_all_variables()
    with tf.Session() as session:
        session.run(init)

        for i in range(max_iter):
            for j in range(n_batches):
                Xbatch = X[j*batch_sz:(j*batch_sz + batch_sz),]
                Ybatch = Y[j*batch_sz:(j*batch_sz + batch_sz),]

                session.run(train_op, feed_dict={self.inputs: Xbatch, self.targets: Ybatch})
                if j % 200 == 0:
                    test_cost = session.run(cost, feed_dict={self.inputs: Xtest, self.targets: Ytest})
                    Ptest = session.run(self.predict_op, feed_dict={self.inputs: Xtest})
                    err = error_rate(Ptest, Ytest)
                    print("Cost / err at iteration i=%d, j=%d: %.3f / %.3f" % (i, j, test_cost, err))
                    costs.append(test_cost)

        # save the model
        self.saver.save(session, self.savefile)

    # save dimensions for later
    self.D = D
    self.K = K

    plt.plot(costs)
    plt.show()
//  [python]
def fit(self, X, Y, Xtest, Ytest):
    N, D = X.shape
    K = len(set(Y))

    # hyperparams
    max_iter = 30
    lr = 1e-3
    mu = 0.9
    regularization = 1e-1
    batch_sz = 100
    n_batches = N // batch_sz

    cost = self.build(D, K)
    l2_penalty = regularization*tf.reduce_mean(self.W**2) / 2
    cost += l2_penalty
    train_op = tf.train.MomentumOptimizer(lr, momentum=mu).minimize(cost)

    costs = []
    init = tf.global_variables_initializer()
    with tf.Session() as session:
        session.run(init)

        for i in range(max_iter):
            for j in range(n_batches):
                Xbatch = X[j*batch_sz:(j*batch_sz + batch_sz),]
                Ybatch = Y[j*batch_sz:(j*batch_sz + batch_sz),]

                session.run(train_op, feed_dict={self.inputs: Xbatch, self.targets: Ybatch})
                if j % 200 == 0:
                    test_cost = session.run(cost, feed_dict={self.inputs: Xtest, self.targets: Ytest})
                    Ptest = session.run(self.predict_op, feed_dict={self.inputs: Xtest})
                    err = error_rate(Ptest, Ytest)
                    print("Cost / err at iteration i=%d, j=%d: %.3f / %.3f" % (i, j, test_cost, err))
                    costs.append(test_cost)

        # save the model
        self.saver.save(session, self.savefile)

    # save dimensions for later
    self.D = D
    self.K = K

    plt.plot(costs)
    plt.show()
//  [python]
def testUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 2
    height, width = 299, 299
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = inception.inception_v3(inputs, num_classes)
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Mixed_7c']
      feed_dict = {inputs: input_np}
      tf.initialize_all_variables().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 8, 2048])
//  [python]
def testUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 2
    height, width = 299, 299
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = inception.inception_v3(inputs, num_classes)
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Mixed_7c']
      feed_dict = {inputs: input_np}
      tf.global_variables_initializer().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 8, 2048])
//  [python]
def testUnknowBatchSize(self):
    batch_size = 1
    height, width = 299, 299
    num_classes = 1000

    inputs = tf.placeholder(tf.float32, (None, height, width, 3))
    logits, _ = inception.inception_v3(inputs, num_classes)
    self.assertTrue(logits.op.name.startswith('InceptionV3/Logits'))
    self.assertListEqual(logits.get_shape().as_list(),
                         [None, num_classes])
    images = tf.random_uniform((batch_size, height, width, 3))

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch_size, num_classes))
//  [python]
def testUnknowBatchSize(self):
    batch_size = 1
    height, width = 299, 299
    num_classes = 1000

    inputs = tf.placeholder(tf.float32, (None, height, width, 3))
    logits, _ = inception.inception_v3(inputs, num_classes)
    self.assertTrue(logits.op.name.startswith('InceptionV3/Logits'))
    self.assertListEqual(logits.get_shape().as_list(),
                         [None, num_classes])
    images = tf.random_uniform((batch_size, height, width, 3))

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch_size, num_classes))
//  [python]
def testGlobalPoolUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 1
    height, width = 330, 400
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = inception.inception_v3(inputs, num_classes,
                                                  global_pool=True)
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Mixed_7c']
      feed_dict = {inputs: input_np}
      tf.initialize_all_variables().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 11, 2048])
//  [python]
def testGlobalPoolUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 1
    height, width = 330, 400
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = inception.inception_v3(inputs, num_classes,
                                                  global_pool=True)
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Mixed_7c']
      feed_dict = {inputs: input_np}
      tf.global_variables_initializer().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 11, 2048])
//  [python]
def testEvaluation(self):
    batch_size = 2
    height, width = 299, 299
    num_classes = 1000

    eval_inputs = tf.random_uniform((batch_size, height, width, 3))
    logits, _ = inception.inception_v3(eval_inputs, num_classes,
                                       is_training=False)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def testEvaluation(self):
    batch_size = 2
    height, width = 299, 299
    num_classes = 1000

    eval_inputs = tf.random_uniform((batch_size, height, width, 3))
    logits, _ = inception.inception_v3(eval_inputs, num_classes,
                                       is_training=False)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def initialize_tf_variables():
    """Initializes tensorflow global/local variables.
    This is needed for custom metrics.
    """
    tf.compat.v1.global_variables_initializer().run()
    tf.compat.v1.local_variables_initializer().run()
//  [python]
def initialize_tf_variables():
    """Initializes tensorflow global/local variables.
    This is needed for custom metrics.
    """
    K.get_session().run(tf.global_variables_initializer())
    K.get_session().run(tf.local_variables_initializer())
//  [python]
def testLogitsNotSqueezed(self):
    num_classes = 25
    images = tf.random_uniform([1, 299, 299, 3])
    logits, _ = inception.inception_v3(images,
                                       num_classes=num_classes,
                                       spatial_squeeze=False)

    with self.test_session() as sess:
      tf.global_variables_initializer().run()
      logits_out = sess.run(logits)
      self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes])
//  [python]
def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000

    train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
    inception.inception_v3(train_inputs, num_classes)
    eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
    logits, _ = inception.inception_v3(eval_inputs, num_classes,
                                       is_training=False, reuse=True)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,))
//  [python]
def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000

    train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
    inception.inception_v3(train_inputs, num_classes)
    eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
    logits, _ = inception.inception_v3(eval_inputs, num_classes,
                                       is_training=False, reuse=True)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,))
//  [python]
def _createTestTrainModel(self, m_creator, hparams, sess):
    train_mode = tf.contrib.learn.ModeKeys.TRAIN
    train_iterator, src_vocab_table, tgt_vocab_table = common_test_utils.create_test_iterator(
        hparams, train_mode)
    train_m = m_creator(
        hparams,
        train_mode,
        train_iterator,
        src_vocab_table,
        tgt_vocab_table,
        scope='dynamic_seq2seq')
    sess.run(tf.global_variables_initializer())
    sess.run(tf.tables_initializer())
    sess.run(train_iterator.initializer)
    return train_m