jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_predictor_builder(self):
    predictor_text_proto = """
    attention_predictor {
      rnn_cell {
        lstm_cell {
          num_units: 256
          forget_bias: 1.0
          initializer { orthogonal_initializer { } }
        }
      }
      rnn_regularizer { l2_regularizer { weight: 1e-4 } }
      num_attention_units: 128
      max_num_steps: 10
      multi_attention: false
      beam_width: 1
      reverse: false
      label_map {
        character_set {
          text_string: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
          delimiter: ""
        }
        label_offset: 2
      }
      loss {
        sequence_cross_entropy_loss {
          sequence_normalize: false
          sample_normalize: true
        }
      }
    }
    """
    predictor_proto = predictor_pb2.Predictor()
    text_format.Merge(predictor_text_proto, predictor_proto)
    predictor_object = predictor_builder.build(predictor_proto, True)

    feature_maps = [tf.random_uniform([2, 1, 10, 32], dtype=tf.float32)]
    predictor_object.provide_groundtruth(
      tf.constant([b'hello', b'world'], dtype=tf.string)
    )
    predictions_dict = predictor_object.predict(feature_maps)
    loss = predictor_object.loss(predictions_dict)

    with self.test_session() as sess:
      sess.run([
        tf.global_variables_initializer(),
        tf.tables_initializer()])
      sess_outputs = sess.run({'loss': loss})
      print(sess_outputs)
//  [python]
def _run_and_test(self,
                      hparams,
                      test_batch_size=False,
                      length_inc=None):
        # Construct database
        text_data = tx.data.MonoTextData(hparams)
        self.assertEqual(text_data.vocab.size,
                         self._vocab_size + len(text_data.vocab.special_tokens))

        iterator = text_data.dataset.make_initializable_iterator()
        text_data_batch = iterator.get_next()

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())
            sess.run(iterator.initializer)

            while True:
                try:
                    data_batch_ = sess.run(text_data_batch)

                    self.assertEqual(set(data_batch_.keys()),
                                     set(text_data.list_items()))

                    if test_batch_size:
                        self.assertEqual(len(data_batch_['text']),
                                         hparams['batch_size'])

                    if length_inc:
                        for i in range(len(data_batch_['text'])):
                            text_ = data_batch_['text'][i].tolist()
                            self.assertEqual(
                                text_.index(b'<EOS>') + 1,
                                data_batch_['length'][i] - length_inc)

                    max_seq_length = text_data.hparams.dataset.max_seq_length
                    mode = text_data.hparams.dataset.length_filter_mode
                    if max_seq_length == 6:
                        max_l = max_seq_length
                        max_l += text_data._decoder.added_length
                        for length in data_batch_['length']:
                            self.assertLessEqual(length, max_l)
                        if mode == "discard":
                            for length in data_batch_['length']:
                                self.assertEqual(length, 5)
                        elif mode == "truncate":
                            num_length_6 = 0
                            for length in data_batch_['length']:
                                num_length_6 += int(length == 6)
                            self.assertGreater(num_length_6, 0)
                        else:
                            raise ValueError("Unknown mode: %s" % mode)

                    if text_data.hparams.dataset.pad_to_max_seq_length:
                        max_l = max_seq_length + text_data._decoder.added_length
                        for x in data_batch_['text']:
                            self.assertEqual(len(x), max_l)
                        for x in data_batch_['text_ids']:
                            self.assertEqual(len(x), max_l)

                except tf.errors.OutOfRangeError:
                    print('Done -- epoch limit reached')
                    break
//  [python]
def train(config, logger, train_and_valid_data):
    if config.do_train_visualized:  # loss可视化
        from tensorboardX import SummaryWriter
        train_writer = SummaryWriter(config.log_save_path + "Train")
        eval_writer = SummaryWriter(config.log_save_path + "Eval")

    with tf.variable_scope("stock_predict"):
        model = Model(config)

    train_X, train_Y, valid_X, valid_Y = train_and_valid_data
    train_len = len(train_X)
    valid_len = len(valid_X)

    if config.use_cuda:     # 开启GPU训练会有很多警告，但不影响训练
        sess_config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True) #在CUDA可用时会自动选择GPU，否则CPU
        sess_config.gpu_options.per_process_gpu_memory_fraction = 0.7  # 显存占用率
        sess_config.gpu_options.allow_growth=True   #初始化时不全部占满GPU显存, 按需分配
    else:
        sess_config = None

    with tf.Session(config=sess_config) as sess:
        sess.run(tf.initialize_all_variables())

        valid_loss_min = float("inf")
        bad_epoch = 0
        global_step = 0
        for epoch in range(config.epoch):
            logger.info("Epoch {}/{}".format(epoch, config.epoch))
            # 训练
            train_loss_array = []
            for step in range(train_len//config.batch_size):
                feed_dict = {model.X: train_X[step*config.batch_size : (step+1)*config.batch_size],
                             model.Y: train_Y[step*config.batch_size : (step+1)*config.batch_size]}
                train_loss, _ = sess.run([model.loss,model.optim], feed_dict=feed_dict)
                train_loss_array.append(train_loss)
                if config.do_train_visualized and global_step % 100 == 0:   # 每一百步显示一次
                    train_writer.add_scalar('Train_Loss', train_loss, global_step+1)
                global_step += 1

            # 验证与早停
            valid_loss_array = []
            for step in range(valid_len//config.batch_size):
                feed_dict = {model.X: valid_X[step * config.batch_size: (step + 1) * config.batch_size],
                             model.Y: valid_Y[step * config.batch_size: (step + 1) * config.batch_size]}
                valid_loss = sess.run(model.loss, feed_dict=feed_dict)
                valid_loss_array.append(valid_loss)

            train_loss_cur = np.mean(train_loss_array)
            valid_loss_cur = np.mean(valid_loss_array)
            logger.info("The train loss is {:.6f}. ".format(train_loss_cur) +
                        "The valid loss is {:.6f}.".format(valid_loss_cur))
            if config.do_train_visualized:
                train_writer.add_scalar('Epoch_Loss', train_loss_cur, epoch + 1)
                eval_writer.add_scalar('Epoch_Loss', valid_loss_cur, epoch + 1)

            if valid_loss_cur < valid_loss_min:
                valid_loss_min = valid_loss_cur
                bad_epoch = 0
                model.saver.save(sess, config.model_save_path + config.model_name)
            else:
                bad_epoch += 1
                if bad_epoch >= config.patience:
                    logger.info(" The training stops early in epoch {}".format(epoch))
                    break
//  [python]
def train(config, logger, train_and_valid_data):
    if config.do_train_visualized:  # loss可视化
        from tensorboardX import SummaryWriter
        train_writer = SummaryWriter(config.log_save_path + "Train")
        eval_writer = SummaryWriter(config.log_save_path + "Eval")

    with tf.variable_scope("stock_predict"):
        model = Model(config)

    train_X, train_Y, valid_X, valid_Y = train_and_valid_data
    train_len = len(train_X)
    valid_len = len(valid_X)

    if config.use_cuda:     # 开启GPU训练会有很多警告，但不影响训练
        sess_config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True) #在CUDA可用时会自动选择GPU，否则CPU
        sess_config.gpu_options.per_process_gpu_memory_fraction = 0.7  # 显存占用率
        sess_config.gpu_options.allow_growth=True   #初始化时不全部占满GPU显存, 按需分配
    else:
        sess_config = None

    with tf.Session(config=sess_config) as sess:
        sess.run(tf.global_variables_initializer())

        valid_loss_min = float("inf")
        bad_epoch = 0
        global_step = 0
        for epoch in range(config.epoch):
            logger.info("Epoch {}/{}".format(epoch, config.epoch))
            # 训练
            train_loss_array = []
            for step in range(train_len//config.batch_size):
                feed_dict = {model.X: train_X[step*config.batch_size : (step+1)*config.batch_size],
                             model.Y: train_Y[step*config.batch_size : (step+1)*config.batch_size]}
                train_loss, _ = sess.run([model.loss,model.optim], feed_dict=feed_dict)
                train_loss_array.append(train_loss)
                if config.do_train_visualized and global_step % 100 == 0:   # 每一百步显示一次
                    train_writer.add_scalar('Train_Loss', train_loss, global_step+1)
                global_step += 1

            # 验证与早停
            valid_loss_array = []
            for step in range(valid_len//config.batch_size):
                feed_dict = {model.X: valid_X[step * config.batch_size: (step + 1) * config.batch_size],
                             model.Y: valid_Y[step * config.batch_size: (step + 1) * config.batch_size]}
                valid_loss = sess.run(model.loss, feed_dict=feed_dict)
                valid_loss_array.append(valid_loss)

            train_loss_cur = np.mean(train_loss_array)
            valid_loss_cur = np.mean(valid_loss_array)
            logger.info("The train loss is {:.6f}. ".format(train_loss_cur) +
                        "The valid loss is {:.6f}.".format(valid_loss_cur))
            if config.do_train_visualized:
                train_writer.add_scalar('Epoch_Loss', train_loss_cur, epoch + 1)
                eval_writer.add_scalar('Epoch_Loss', valid_loss_cur, epoch + 1)

            if valid_loss_cur < valid_loss_min:
                valid_loss_min = valid_loss_cur
                bad_epoch = 0
                model.saver.save(sess, config.model_save_path + config.model_name)
            else:
                bad_epoch += 1
                if bad_epoch >= config.patience:
                    logger.info(" The training stops early in epoch {}".format(epoch))
                    break
//  [python]
def test_one(self):
        """Load all images from a folder once and print the result."""
        # Create the queue operations
        image_key, image_tensor, image_shape = queue_single_images_from_folder(
            DIR_RESIZED
        )

        # Start a new session to run the operations
        with tf.Session() as sess:
            sess.run(tf.initialize_all_variables())
            sess.run(tf.local_variables_initializer())

            # Coordinate the loading of image files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)
            count = 0
            start_time = time.time()

            # These are the only lines where something happens:
            # we execute the operations to get one image and print everything.
            try:
                while not coord.should_stop():
                    key, img, shape = sess.run([image_key, image_tensor, image_shape])
                    print(key)
                    count += 1
            except tf.errors.OutOfRangeError:
                # It's all right, it's just the string_input_producer queue telling
                # us that it has run out of strings
                pass
            finally:
                # Ask the threads (filename queue) to stop.
                coord.request_stop()
                print(
                    "Finished listing {} pairs in {:.2f}s".format(
                        count, time.time() - start_time
                    )
                )

            # Wait for threads to finish.
            coord.join(threads)
//  [python]
def test_one(self):
        """Load all images from a folder once and print the result."""
        # Create the queue operations
        image_key, image_tensor, image_shape = queue_single_images_from_folder(
            DIR_RESIZED
        )

        # Start a new session to run the operations
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())

            # Coordinate the loading of image files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)
            count = 0
            start_time = time.time()

            # These are the only lines where something happens:
            # we execute the operations to get one image and print everything.
            try:
                while not coord.should_stop():
                    key, img, shape = sess.run([image_key, image_tensor, image_shape])
                    print(key)
                    count += 1
            except tf.errors.OutOfRangeError:
                # It's all right, it's just the string_input_producer queue telling
                # us that it has run out of strings
                pass
            finally:
                # Ask the threads (filename queue) to stop.
                coord.request_stop()
                print(
                    "Finished listing {} pairs in {:.2f}s".format(
                        count, time.time() - start_time
                    )
                )

            # Wait for threads to finish.
            coord.join(threads)
//  [python]
def test_sync_predictor_builder(self):
    predictor_text_proto = """
    attention_predictor {
      rnn_cell {
        lstm_cell {
          num_units: 256
          forget_bias: 1.0
          initializer { orthogonal_initializer { } }
        }
      }
      rnn_regularizer { l2_regularizer { weight: 1e-4 } }
      num_attention_units: 128
      max_num_steps: 10
      multi_attention: false
      beam_width: 1
      reverse: false
      label_map {
        character_set {
          text_string: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
          delimiter: ""
        }
        label_offset: 2
      }
      loss {
        sequence_cross_entropy_loss {
          sequence_normalize: false
          sample_normalize: true
        }
      }
      sync: true
    }
    """
    predictor_proto = predictor_pb2.Predictor()
    text_format.Merge(predictor_text_proto, predictor_proto)
    predictor_object = predictor_builder.build(predictor_proto, True)

    feature_maps = [tf.random_uniform([2, 1, 10, 32], dtype=tf.float32)]
    predictor_object.provide_groundtruth(
      tf.constant([b'hello', b'world'], dtype=tf.string)
    )
    predictions_dict = predictor_object.predict(feature_maps)
    loss = predictor_object.loss(predictions_dict)

    with self.test_session() as sess:
      sess.run([
        tf.initialize_all_variables(),
        tf.tables_initializer()])
      sess_outputs = sess.run({'loss': loss})
      print(sess_outputs)
//  [python]
def test_sync_predictor_builder(self):
    predictor_text_proto = """
    attention_predictor {
      rnn_cell {
        lstm_cell {
          num_units: 256
          forget_bias: 1.0
          initializer { orthogonal_initializer { } }
        }
      }
      rnn_regularizer { l2_regularizer { weight: 1e-4 } }
      num_attention_units: 128
      max_num_steps: 10
      multi_attention: false
      beam_width: 1
      reverse: false
      label_map {
        character_set {
          text_string: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
          delimiter: ""
        }
        label_offset: 2
      }
      loss {
        sequence_cross_entropy_loss {
          sequence_normalize: false
          sample_normalize: true
        }
      }
      sync: true
    }
    """
    predictor_proto = predictor_pb2.Predictor()
    text_format.Merge(predictor_text_proto, predictor_proto)
    predictor_object = predictor_builder.build(predictor_proto, True)

    feature_maps = [tf.random_uniform([2, 1, 10, 32], dtype=tf.float32)]
    predictor_object.provide_groundtruth(
      tf.constant([b'hello', b'world'], dtype=tf.string)
    )
    predictions_dict = predictor_object.predict(feature_maps)
    loss = predictor_object.loss(predictions_dict)

    with self.test_session() as sess:
      sess.run([
        tf.global_variables_initializer(),
        tf.tables_initializer()])
      sess_outputs = sess.run({'loss': loss})
      print(sess_outputs)
//  [python]
def graph_save(pred, config):
    """
    run classify predict
    """
    graph_path=config["graph_path"]
    graph_name=config["graph_name"]
    mode = config["training_mode"]
    if mode == "pointwise":
        pred_prob = tf.nn.softmax(pred, -1, name="output_prob")
    elif mode == "pairwise":
        pred_prob = tf.identity(pred, name="output_prob")
    saver = tf.train.Saver()
    step = 0
    init = tf.group(tf.global_variables_initializer(),
                    tf.local_variables_initializer())
    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1)) \
                    as sess:
        sess.run(init)
        tf.train.write_graph(sess.graph_def, graph_path, graph_name, as_text=True)
    sess.close()
//  [python]
def run_predict(pred, label, config):
    """
    run classification predict function handle
    """
    mean_acc = 0.0
    saver = tf.train.Saver()
    mode = config["training_mode"]
    label_index = tf.argmax(label, 1)
    if mode == "pointwise":
        pred_prob = tf.nn.softmax(pred, -1)
        score = tf.reduce_max(pred_prob, -1)
        pred_index = tf.argmax(pred_prob, 1)
        correct_pred = tf.equal(pred_index, label_index)
        acc = tf.reduce_mean(tf.cast(correct_pred, "float"))
    elif mode == "pairwise":
        score = pred
        pred_index = tf.argmax(pred, 1)
        acc = tf.constant([0.0])
    modelfile = config["test_model_file"]
    result_file = file(config["test_result"], "w")
    step = 0
    init = tf.group(tf.initialize_all_variables(),
                    tf.local_variables_initializer())
    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1)) \
                    as sess:
        sess.run(init)
        saver.restore(sess, modelfile)
        coord = tf.train.Coordinator()
        read_thread = tf.train.start_queue_runners(sess=sess, coord=coord)
        while not coord.should_stop():
            step += 1
            try:
                ground, pi, a, prob = sess.run([label_index, pred_index, acc, score])
                mean_acc += a
                for i in range(len(prob)):
                    result_file.write("%d\t%d\t%f\n" % (ground[i], pi[i], prob[i]))
            except tf.errors.OutOfRangeError:
                coord.request_stop()
        coord.join(read_thread)
    sess.close()
    result_file.close()
    if mode == "pointwise":
        mean_acc = mean_acc / step
        print >> sys.stderr, "accuracy: %4.2f" % (mean_acc * 100)
//  [python]
def run_predict(pred, label, config):
    """
    run classification predict function handle
    """
    mean_acc = 0.0
    saver = tf.train.Saver()
    mode = config["training_mode"]
    label_index = tf.argmax(label, 1)
    if mode == "pointwise":
        pred_prob = tf.nn.softmax(pred, -1)
        score = tf.reduce_max(pred_prob, -1)
        pred_index = tf.argmax(pred_prob, 1)
        correct_pred = tf.equal(pred_index, label_index)
        acc = tf.reduce_mean(tf.cast(correct_pred, "float"))
    elif mode == "pairwise":
        score = pred
        pred_index = tf.argmax(pred, 1)
        acc = tf.constant([0.0])
    modelfile = config["test_model_file"]
    result_file = file(config["test_result"], "w")
    step = 0
    init = tf.group(tf.global_variables_initializer(),
                    tf.local_variables_initializer())
    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1)) \
                    as sess:
        sess.run(init)
        saver.restore(sess, modelfile)
        coord = tf.train.Coordinator()
        read_thread = tf.train.start_queue_runners(sess=sess, coord=coord)
        while not coord.should_stop():
            step += 1
            try:
                ground, pi, a, prob = sess.run([label_index, pred_index, acc, score])
                mean_acc += a
                for i in range(len(prob)):
                    result_file.write("%d\t%d\t%f\n" % (ground[i], pi[i], prob[i]))
            except tf.errors.OutOfRangeError:
                coord.request_stop()
        coord.join(read_thread)
    sess.close()
    result_file.close()
    if mode == "pointwise":
        mean_acc = mean_acc / step
        print >> sys.stderr, "accuracy: %4.2f" % (mean_acc * 100)
//  [python]
def _checkOutputs(self, expected_shapes, feed_dict=None):
    """Verifies that the model produces expected outputs.

    Args:
      expected_shapes: A dict mapping Tensor or Tensor name to expected output
        shape.
      feed_dict: Values of Tensors to feed into Session.run().
    """
    fetches = expected_shapes.keys()

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      outputs = sess.run(fetches, feed_dict)

    for index, output in enumerate(outputs):
      tensor = fetches[index]
      expected = expected_shapes[tensor]
      actual = output.shape
      if expected != actual:
        self.fail("Tensor %s has shape %s (expected %s)." % (tensor, actual,
                                                             expected))
//  [python]
def run_trainer(loss, optimizer, config):
    """
    run classification training function handle
    """
    thread_num = int(config["thread_num"])
    model_path = config["model_path"]
    model_file = config["model_prefix"]
    print_iter = int(config["print_iter"])
    data_size = int(config["data_size"])
    batch_size = int(config["batch_size"])
    epoch_iter = int(data_size / batch_size)
    avg_cost = 0.0
    saver = tf.train.Saver(max_to_keep=None)
    init = tf.group(tf.initialize_all_variables(),
                    tf.local_variables_initializer())
    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=thread_num, 
                                          inter_op_parallelism_threads=thread_num)) \
                    as sess:
        sess.run(init)
        coord = tf.train.Coordinator()
        read_thread = tf.train.start_queue_runners(sess=sess, coord=coord)
        step = 0
        epoch_num = 1
        start_time = time.time()
        while not coord.should_stop():
            try:
                step += 1
                c, _= sess.run([loss, optimizer])
                avg_cost += c

                if step % print_iter == 0:
                    print("loss: %f" % ((avg_cost / print_iter)))
                    avg_cost = 0.0
                if step % epoch_iter == 0:
                    end_time = time.time()
                    print("save model epoch%d, used time: %d" % (epoch_num, 
                          end_time - start_time))
                    save_path = saver.save(sess, 
                            "%s/%s.epoch%d" % (model_path, model_file, epoch_num))
                    epoch_num += 1
                    start_time = time.time()
                    
            except tf.errors.OutOfRangeError:
                save_path = saver.save(sess, "%s/%s.final" % (model_path, model_file))
                coord.request_stop()
        coord.join(read_thread)
    sess.close()
//  [python]
def run_trainer(loss, optimizer, config):
    """
    run classification training function handle
    """
    thread_num = int(config["thread_num"])
    model_path = config["model_path"]
    model_file = config["model_prefix"]
    print_iter = int(config["print_iter"])
    data_size = int(config["data_size"])
    batch_size = int(config["batch_size"])
    epoch_iter = int(data_size / batch_size)
    avg_cost = 0.0
    saver = tf.train.Saver(max_to_keep=None)
    init = tf.group(tf.global_variables_initializer(),
                    tf.local_variables_initializer())
    with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=thread_num, 
                                          inter_op_parallelism_threads=thread_num)) \
                    as sess:
        sess.run(init)
        coord = tf.train.Coordinator()
        read_thread = tf.train.start_queue_runners(sess=sess, coord=coord)
        step = 0
        epoch_num = 1
        start_time = time.time()
        while not coord.should_stop():
            try:
                step += 1
                c, _= sess.run([loss, optimizer])
                avg_cost += c

                if step % print_iter == 0:
                    print("loss: %f" % ((avg_cost / print_iter)))
                    avg_cost = 0.0
                if step % epoch_iter == 0:
                    end_time = time.time()
                    print("save model epoch%d, used time: %d" % (epoch_num, 
                          end_time - start_time))
                    save_path = saver.save(sess, 
                            "%s/%s.epoch%d" % (model_path, model_file, epoch_num))
                    epoch_num += 1
                    start_time = time.time()
                    
            except tf.errors.OutOfRangeError:
                save_path = saver.save(sess, "%s/%s.final" % (model_path, model_file))
                coord.request_stop()
        coord.join(read_thread)
    sess.close()
//  [python]
def train_network(num_epochs,num_steps,state_size=4,verbose=True):
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        training_losses = []
        for idx,epoch in enumerate(gen_epochs(num_epochs)):
            training_loss = 0
            training_state = np.zeros((batch_size,state_size))
            if verbose:
                print("\n EPOCH",idx)
            for step,(X,Y) in enumerate(epoch):
                tr_losses,training_loss_,training_state,_ = sess.run([loss,total_loss,final_state,train_step],
                                                                     feed_dict={x:X,y:Y,init_state:training_state})
                training_loss += training_loss_

                if step % 100 == 0 and step > 0:
                    if verbose:
                        print("Average loss at step",step,"for last 100 steps:",training_loss/100)
                    training_losses.append(training_loss/100)
                training_loss = 0


        return training_losses
//  [python]
def train_network(num_epochs,num_steps,state_size=4,verbose=True):
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        training_losses = []
        for idx,epoch in enumerate(gen_epochs(num_epochs)):
            training_loss = 0
            training_state = np.zeros((batch_size,state_size))
            if verbose:
                print("\n EPOCH",idx)
            for step,(X,Y) in enumerate(epoch):
                tr_losses,training_loss_,training_state,_ = sess.run([loss,total_loss,final_state,train_step],
                                                                     feed_dict={x:X,y:Y,init_state:training_state})
                training_loss += training_loss_

                if step % 100 == 0 and step > 0:
                    if verbose:
                        print("Average loss at step",step,"for last 100 steps:",training_loss/100)
                    training_losses.append(training_loss/100)
                training_loss = 0


        return training_losses
//  [python]
def tfrecord2npy(path):

    parser = S2parser()

    def mapping_function(serialized_feature):
        # read data from .tfrecords
        feature = parser.parse_example(serialized_example=serialized_feature)
        return feature

    dataset = tf.data.TFRecordDataset([path], compression_type="GZIP")

    dataset = dataset.map(mapping_function, num_parallel_calls=1)
    sess = tf.InteractiveSession()
    sess.run([tf.global_variables_initializer(), tf.local_variables_initializer(), tf.tables_initializer()])
    iterator = dataset.make_initializable_iterator()
    sess.run([iterator.initializer])
    x10,x20,x60,doy,year,labels = sess.run(iterator.get_next())

    x10, x20, x60, doy, year, labels = [np.array(f) for f in [x10, x20, x60, doy, year, labels]]

    x10, x20, x60, doy, year, labels = remove_padded_instances(x10, x20, x60, doy, year, labels)

    return x10, x20, x60, doy, year, labels
//  [python]
def _lab_image_write(self):
        # Create the queue operations
        img_key, img_tensor, _ = queue_single_images_from_folder(DIR_RESIZED)
        img_emb = tf.truncated_normal(shape=[1001])

        # Create a writer to write_image the images
        lab_writer = LabImageRecordWriter("test_lab_images.tfrecord", DIR_TFRECORDS)

        # Start a new session to run the operations
        with tf.Session() as sess:
            sess.run(
                [tf.global_variables_initializer(), tf.local_variables_initializer()]
            )

            # Coordinate the loading of image files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            count = 0
            start_time = time.time()
            try:
                while not coord.should_stop():
                    key, img, emb = sess.run([img_key, img_tensor, img_emb])
                    lab_writer.write_image(key, img, emb)
                    print("Written: {}".format(key))
                    count += 1
                    # Just write 10 images
                    if count > 10:
                        break
            except tf.errors.OutOfRangeError:
                # The string_input_producer queue ran out of strings
                pass
            finally:
                # Ask the threads (filename queue) to stop.
                coord.request_stop()
                print(
                    "Finished writing {} images in {:.2f}s".format(
                        count, time.time() - start_time
                    )
                )

            # Wait for threads to finish.
            coord.join(threads)

        lab_writer.close()
//  [python]
def _lab_image_read(self):
        # Important: read_batch MUST be called before start_queue_runners,
        # otherwise the internal shuffle queue gets created but its
        # threads won't start
        irr = LabImageRecordReader("test_lab_images.tfrecord", DIR_TFRECORDS)
        read_one_example = irr.read_operation
        read_batched_examples = irr.read_batch(20)

        with tf.Session() as sess:
            sess.run(
                [tf.global_variables_initializer(), tf.local_variables_initializer()]
            )

            # Coordinate the loading of image files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            # Reading images sequentially one by one
            for i in range(0, 12, 2):
                res = sess.run(read_one_example)
                img = lab_to_rgb(res["image_l"], res["image_ab"])
                img_gray = l_to_rgb(res["image_l"])
                plt.subplot(3, 4, i + 1)
                plt.imshow(img_gray)
                plt.axis("off")
                plt.subplot(3, 4, i + 2)
                plt.imshow(img)
                plt.axis("off")
                print("Read", basename(res["image_name"]))
            plt.show()

            # Reading images in batch
            res = sess.run(read_batched_examples)
            print(
                res["image_name"],
                res["image_l"].shape,
                res["image_ab"].shape,
                res["image_embedding"].shape,
                sep="\n",
            )

            # Finish off the filename queue coordinator.
            coord.request_stop()
            coord.join(threads)
//  [python]
def restore(self, session, step='max'):
    # Don't try to restore unused variables from the TF-Hub ELMo module.
    vars_to_restore = [v for v in tf.global_variables() ]
    saver = tf.train.Saver(vars_to_restore)
    if step in ['max', 'pretrained']:
      path = "model." + step + ".ckpt"
    else:
      path = "model-" + step
    if self.config["restore_path"] != "":
      checkpoint_path = os.path.join(os.path.split(self.config["log_dir"])[0], self.config["restore_path"], path)
    else:
      checkpoint_path = os.path.join(self.config["log_dir"], path)
    print("Restoring from {}".format(checkpoint_path))
    session.run(tf.initialize_all_variables())
    saver.restore(session, checkpoint_path)
//  [python]
def restore(self, session, step='max'):
    # Don't try to restore unused variables from the TF-Hub ELMo module.
    vars_to_restore = [v for v in tf.global_variables() ]
    saver = tf.train.Saver(vars_to_restore)
    if step in ['max', 'pretrained']:
      path = "model." + step + ".ckpt"
    else:
      path = "model-" + step
    if self.config["restore_path"] != "":
      checkpoint_path = os.path.join(os.path.split(self.config["log_dir"])[0], self.config["restore_path"], path)
    else:
      checkpoint_path = os.path.join(self.config["log_dir"], path)
    print("Restoring from {}".format(checkpoint_path))
    session.run(tf.global_variables_initializer())
    saver.restore(session, checkpoint_path)
//  [python]
def train_keywords(self):
        batch = self.iterator.get_next()
        loss, acc, _ = self.predict_keywords(batch)
        op_step = tf.Variable(0, name='op_step')
        train_op = tx.core.get_train_op(loss, global_step=op_step, hparams=self.config.neural_opt_hparams)
        max_val_acc = 0.
        self.saver = tf.train.Saver()
        with tf.Session(config=self.gpu_config) as sess:
            sess.run(tf.initialize_all_variables())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())
            for epoch_id in range(self.config._max_epoch):
                self.iterator.switch_to_train_data(sess)
                cur_step = 0
                cnt_acc = []
                while True:
                    try:
                        cur_step += 1
                        feed = {tx.global_mode(): tf.estimator.ModeKeys.TRAIN}
                        loss_, acc_ = sess.run([train_op, acc], feed_dict=feed)
                        cnt_acc.append(acc_)
                        if cur_step % 200 == 0:
                            print('batch {}, loss={}, acc1={}'.format(cur_step, loss_, np.mean(cnt_acc[-200:])))
                    except tf.errors.OutOfRangeError:
                        break
                self.iterator.switch_to_val_data(sess)
                cnt_acc = []
                while True:
                    try:
                        feed = {tx.global_mode(): tf.estimator.ModeKeys.EVAL}
                        acc_ = sess.run(acc, feed_dict=feed)
                        cnt_acc.append(acc_)
                    except tf.errors.OutOfRangeError:
                        mean_acc = np.mean(cnt_acc)
                        if mean_acc > max_val_acc:
                            max_val_acc = mean_acc
                            self.saver.save(sess, self.config._neural_save_path)
                        print('epoch_id {}, valid acc1={}'.format(epoch_id+1, mean_acc))
                        break
//  [python]
def train_keywords(self):
        batch = self.iterator.get_next()
        loss, acc, _ = self.predict_keywords(batch)
        op_step = tf.Variable(0, name='op_step')
        train_op = tx.core.get_train_op(loss, global_step=op_step, hparams=self.config.neural_opt_hparams)
        max_val_acc = 0.
        self.saver = tf.train.Saver()
        with tf.Session(config=self.gpu_config) as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())
            for epoch_id in range(self.config._max_epoch):
                self.iterator.switch_to_train_data(sess)
                cur_step = 0
                cnt_acc = []
                while True:
                    try:
                        cur_step += 1
                        feed = {tx.global_mode(): tf.estimator.ModeKeys.TRAIN}
                        loss_, acc_ = sess.run([train_op, acc], feed_dict=feed)
                        cnt_acc.append(acc_)
                        if cur_step % 200 == 0:
                            print('batch {}, loss={}, acc1={}'.format(cur_step, loss_, np.mean(cnt_acc[-200:])))
                    except tf.errors.OutOfRangeError:
                        break
                self.iterator.switch_to_val_data(sess)
                cnt_acc = []
                while True:
                    try:
                        feed = {tx.global_mode(): tf.estimator.ModeKeys.EVAL}
                        acc_ = sess.run(acc, feed_dict=feed)
                        cnt_acc.append(acc_)
                    except tf.errors.OutOfRangeError:
                        mean_acc = np.mean(cnt_acc)
                        if mean_acc > max_val_acc:
                            max_val_acc = mean_acc
                            self.saver.save(sess, self.config._neural_save_path)
                        print('epoch_id {}, valid acc1={}'.format(epoch_id+1, mean_acc))
                        break
//  [python]
def tf_baseline_conv2d():
    import tensorflow as tf
    import cntk.contrib.crosstalk.crosstalk_tensorflow as crtf
    ci = crtf.instance

    tf.reset_default_graph()

    x = tf.placeholder(tf.float32, [batch_size, seq_len, num_chars, char_emb_dim])
    filter_bank = tf.get_variable("char_filter_bank",
                                  shape=[filter_width, char_emb_dim, num_filters],
                                  dtype=tf.float32)
    bias = tf.get_variable("char_filter_biases", shape=[num_filters], dtype=tf.float32)

    x_reshape = tf.reshape(x, [-1] + x.get_shape().as_list()[-2:])
    char_conv = tf.expand_dims(tf.transpose(tf.nn.conv1d(x_reshape, filter_bank, stride=1, padding='VALID') + bias, perm=[0,2,1]), -1)
    char_conv = tf.reshape(char_conv, [-1, seq_len] + char_conv.shape.as_list()[-3:])

    ci.watch(cstk.Conv2DArgs(W=crtf.find_trainable('char_filter_bank'), b=crtf.find_trainable('char_filter_biases')), 'conv2d', var_type=cstk.Conv2DAttr,
               attr=cstk.Conv2DAttr(filter_shape=(filter_width, char_emb_dim,), num_filters=num_filters))
    ci.watch(char_conv, 'conv2d_out', var_type=crtf.VariableType) # note the output is transposed to NCHW

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        data = {x:input_data}
        ci.set_workdir(workdir)
        ci.set_data(sess, data)
        ci.fetch('conv2d_out', save=True)
        ci.fetch('conv2d', save=True)
        ci.assign('conv2d', load=True)
        assert ci.compare('conv2d_out')
        ci.reset()
        sess.close()
//  [python]
def tf_baseline_conv2d():
    import tensorflow as tf
    import cntk.contrib.crosstalk.crosstalk_tensorflow as crtf
    ci = crtf.instance

    tf.reset_default_graph()

    x = tf.placeholder(tf.float32, [batch_size, seq_len, num_chars, char_emb_dim])
    filter_bank = tf.get_variable("char_filter_bank",
                                  shape=[filter_width, char_emb_dim, num_filters],
                                  dtype=tf.float32)
    bias = tf.get_variable("char_filter_biases", shape=[num_filters], dtype=tf.float32)

    x_reshape = tf.reshape(x, [-1] + x.get_shape().as_list()[-2:])
    char_conv = tf.expand_dims(tf.transpose(tf.nn.conv1d(x_reshape, filter_bank, stride=1, padding='VALID') + bias, perm=[0,2,1]), -1)
    char_conv = tf.reshape(char_conv, [-1, seq_len] + char_conv.shape.as_list()[-3:])

    ci.watch(cstk.Conv2DArgs(W=crtf.find_trainable('char_filter_bank'), b=crtf.find_trainable('char_filter_biases')), 'conv2d', var_type=cstk.Conv2DAttr,
               attr=cstk.Conv2DAttr(filter_shape=(filter_width, char_emb_dim,), num_filters=num_filters))
    ci.watch(char_conv, 'conv2d_out', var_type=crtf.VariableType) # note the output is transposed to NCHW

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        data = {x:input_data}
        ci.set_workdir(workdir)
        ci.set_data(sess, data)
        ci.fetch('conv2d_out', save=True)
        ci.fetch('conv2d', save=True)
        ci.assign('conv2d', load=True)
        assert ci.compare('conv2d_out')
        ci.reset()
        sess.close()
//  [python]
def test_keywords(self):
        batch = self.iterator.get_next()
        loss, acc, kws = self.predict_keywords(batch)
        saver = tf.train.Saver()
        with tf.Session(config=self.gpu_config) as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())
            saver.restore(sess, self.config._neural_save_path)
            self.iterator.switch_to_test_data(sess)
            cnt_acc, cnt_rec1, cnt_rec3, cnt_rec5 = [], [], [], []
            while True:
                try:
                    feed = {tx.global_mode(): tf.estimator.ModeKeys.PREDICT}
                    acc_, kw_ans, kw_labels = sess.run([acc, kws, batch['keywords_text_ids']], feed_dict=feed)
                    cnt_acc.append(acc_)
                    rec = [0,0,0,0,0]
                    sum_kws = 0
                    for i in range(len(kw_ans)):
                        sum_kws += sum(kw_labels[i] > 3)
                        for j in range(5):
                            if kw_ans[i][j] in kw_labels[i]:
                                for k in range(j, 5):
                                    rec[k] += 1
                    cnt_rec1.append(rec[0]/sum_kws)
                    cnt_rec3.append(rec[2]/sum_kws)
                    cnt_rec5.append(rec[4]/sum_kws)

                except tf.errors.OutOfRangeError:
                    print('test_kw acc@1={:.4f}, rec@1={:.4f}, rec@3={:.4f}, rec@5={:.4f}'.format(
                        np.mean(cnt_acc), np.mean(cnt_rec1), np.mean(cnt_rec3), np.mean(cnt_rec5)))
                    break
//  [python]
def init_variables_from_state_dict(sess,
                                   state_dict,
                                   ignore_extra=True):
    """
    Initialize model variables from state dictionary.

    Parameters:
    ----------
    sess: Session
        A Session to use to load the weights.
    state_dict : dict
        Dictionary with values of model variables.
    ignore_extra : bool, default True
        Whether to silently ignore parameters from the file that are not present in this Module.
    """
    import tensorflow as tf
    assert sess is not None
    if state_dict is None:
        raise Exception("The state dict is empty")
    dst_params = {v.name: v for v in tf.global_variables()}
    sess.run(tf.initialize_all_variables())
    for src_key in state_dict.keys():
        if src_key in dst_params.keys():
            assert (state_dict[src_key].shape == tuple(dst_params[src_key].get_shape().as_list()))
            sess.run(dst_params[src_key].assign(state_dict[src_key]))
        elif not ignore_extra:
            raise Exception("The state dict is incompatible with the model")
        else:
            print("Key `{}` is ignored".format(src_key))
//  [python]
def init_variables_from_state_dict(sess,
                                   state_dict,
                                   ignore_extra=True):
    """
    Initialize model variables from state dictionary.

    Parameters:
    ----------
    sess: Session
        A Session to use to load the weights.
    state_dict : dict
        Dictionary with values of model variables.
    ignore_extra : bool, default True
        Whether to silently ignore parameters from the file that are not present in this Module.
    """
    import tensorflow as tf
    assert sess is not None
    if state_dict is None:
        raise Exception("The state dict is empty")
    dst_params = {v.name: v for v in tf.global_variables()}
    sess.run(tf.global_variables_initializer())
    for src_key in state_dict.keys():
        if src_key in dst_params.keys():
            assert (state_dict[src_key].shape == tuple(dst_params[src_key].get_shape().as_list()))
            sess.run(dst_params[src_key].assign(state_dict[src_key]))
        elif not ignore_extra:
            raise Exception("The state dict is incompatible with the model")
        else:
            print("Key `{}` is ignored".format(src_key))
//  [python]
def run_tester(self, tester):
        with self.test_session() as sess:
            ops = tester.create_model()
            init_op = tf.group(tf.global_variables_initializer(),
                               tf.local_variables_initializer())
            sess.run(init_op)
            output_result = sess.run(ops)
            tester.check_output(output_result)

            self.assert_all_tensors_reachable(sess, [init_op, ops])
//  [python]
def set_iterators(self, train_iterator=None, validation_iterator=None, eval_from_input_iterator=None, eval_from_latent_iterator=None, eval_from_placeholder=False, eval_uses_fast_inference=False):
        """ Builds the computational graph to every corresponding given iterator.

        Args:
            train_iterator: Dataset Iterator
            validation_iterator: Dataset Iterator
            eval_iterator: Dataset Iterator
        """
        if train_iterator is not None:
            self._set_train_iterator(train_iterator)

        if validation_iterator is not None:
            self._set_validation_iterator(validation_iterator)

        if eval_from_input_iterator is not None:
            self._set_eval_from_input_iterator(eval_from_input_iterator, eval_uses_fast_inference)

        if eval_from_latent_iterator is not None:
            self._set_eval_from_latent_iterator(eval_from_latent_iterator)
            
        if eval_from_placeholder:
            self._set_eval_from_placeholder()            

        self.sess.run(tf.global_variables_initializer())
//  [python]
def execute_tpu(self, graph_fn, inputs):
        """Constructs the graph, executes it on TPU and returns the result.

        Args:
          graph_fn: a callable that constructs the tensorflow graph to test. The
            arguments of this function should correspond to `inputs`.
          inputs: a list of numpy arrays to feed input to the computation graph.

        Returns:
          A list of numpy arrays or a scalar returned from executing the tensorflow
          graph.
        """
        with self.test_session(graph=tf.Graph()) as sess:
            placeholders = [tf.placeholder_with_default(v, v.shape) for v in inputs]
            tpu_computation = tpu.rewrite(graph_fn, placeholders)
            sess.run(tpu.initialize_system())
            sess.run([tf.global_variables_initializer(), tf.tables_initializer(),
                      tf.local_variables_initializer()])
            materialized_results = sess.run(tpu_computation,
                                            feed_dict=dict(zip(placeholders, inputs)))
            sess.run(tpu.shutdown_system())
            if (len(materialized_results) == 1
                and (isinstance(materialized_results, list)
                     or isinstance(materialized_results, tuple))):
                materialized_results = materialized_results[0]
        return materialized_results