jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test(params):
    """Test function."""

    dataloader = MonodepthDataloader(args.data_path, args.filenames_file, params, args.dataset, args.mode)
    left  = dataloader.left_image_batch
    right = dataloader.right_image_batch

    model = MonodepthModel(params, args.mode, left, right)

    # SESSION
    config = tf.ConfigProto(allow_soft_placement=True)
    sess = tf.Session(config=config)

    # SAVER
    train_saver = tf.train.Saver()

    # INIT
    sess.run(tf.global_variables_initializer())
    sess.run(tf.local_variables_initializer())
    coordinator = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coordinator)

    # RESTORE
    if args.checkpoint_path == '':
        restore_path = tf.train.latest_checkpoint(args.log_directory + '/' + args.model_name)
    else:
        restore_path = args.checkpoint_path.split(".")[0]
    train_saver.restore(sess, restore_path)

    num_test_samples = count_text_lines(args.filenames_file)

    print('now testing {} files'.format(num_test_samples))
    disparities    = np.zeros((num_test_samples, params.height, params.width), dtype=np.float32)
    disparities_pp = np.zeros((num_test_samples, params.height, params.width), dtype=np.float32)
    for step in range(num_test_samples):
        disp = sess.run(model.disp_left_est[0])
        disparities[step] = disp[0].squeeze()
        disparities_pp[step] = post_process_disparity(disp.squeeze())

    print('done.')

    print('writing disparities.')
    if args.output_directory == '':
        output_directory = os.path.dirname(args.checkpoint_path)
    else:
        output_directory = args.output_directory
    np.save(output_directory + '/disparities.npy',    disparities)
    np.save(output_directory + '/disparities_pp.npy', disparities_pp)

    print('done.')
//  [python]
def init_tf_sess(self):
        tf_config = tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)
        tf_config.gpu_options.allow_growth = True # may need if using GPU
        self.sess = tf.Session(config=tf_config)
        self.sess.__enter__() # equivalent to `with self.sess:`
        tf.initialize_all_variables().run()
//  [python]
def init_tf_sess(self):
        tf_config = tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)
        tf_config.gpu_options.allow_growth = True # may need if using GPU
        self.sess = tf.Session(config=tf_config)
        self.sess.__enter__() # equivalent to `with self.sess:`
        tf.global_variables_initializer().run()
//  [python]
def init_session(frac=0.3):
	global sess
	if sess is None:
		config = tf.ConfigProto()
		config.gpu_options.per_process_gpu_memory_fraction = frac
		sess = tf.Session(config=config)
		# init variables
		sess.run(tf.group(tf.initialize_all_variables(),
                      tf.local_variables_initializer()))
//  [python]
def init_session(frac=0.3):
	global sess
	if sess is None:
		config = tf.ConfigProto()
		config.gpu_options.per_process_gpu_memory_fraction = frac
		sess = tf.Session(config=config)
		# init variables
		sess.run(tf.group(tf.global_variables_initializer(),
                      tf.local_variables_initializer()))
//  [python]
def run_training():
  """Train MNIST for a number of steps."""

  # Tell TensorFlow that the model will be built into the default Graph.
  with tf.Graph().as_default():
    # Input images and labels.
    images, labels = inputs(train=True, batch_size=FLAGS.batch_size,
                            num_epochs=FLAGS.num_epochs)

    # Build a Graph that computes predictions from the inference model.
    logits = mnist.inference(images,
                             FLAGS.hidden1,
                             FLAGS.hidden2)

    # Add to the Graph the loss calculation.
    loss = mnist.loss(logits, labels)

    # Add to the Graph operations that train the model.
    train_op = mnist.training(loss, FLAGS.learning_rate)

    # The op for initializing the variables.
    init_op = tf.group(tf.initialize_all_variables(),
                       tf.local_variables_initializer())

    # Create a session for running operations in the Graph.
    sess = tf.Session()

    # Initialize the variables (the trained variables and the
    # epoch counter).
    sess.run(init_op)

    # Start input enqueue threads.
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)

    try:
      step = 0
      while not coord.should_stop():
        start_time = time.time()

        # Run one step of the model.  The return values are
        # the activations from the `train_op` (which is
        # discarded) and the `loss` op.  To inspect the values
        # of your ops or variables, you may include them in
        # the list passed to sess.run() and the value tensors
        # will be returned in the tuple from the call.
        _, loss_value = sess.run([train_op, loss])

        duration = time.time() - start_time

        # Print an overview fairly often.
        if step % 100 == 0:
          print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value,
                                                     duration))
        step += 1
    except tf.errors.OutOfRangeError:
      print('Done training for %d epochs, %d steps.' % (FLAGS.num_epochs, step))
    finally:
      # When done, ask the threads to stop.
      coord.request_stop()

    # Wait for threads to finish.
    coord.join(threads)
    sess.close()
//  [python]
def run_training():
  """Train MNIST for a number of steps."""

  # Tell TensorFlow that the model will be built into the default Graph.
  with tf.Graph().as_default():
    # Input images and labels.
    images, labels = inputs(train=True, batch_size=FLAGS.batch_size,
                            num_epochs=FLAGS.num_epochs)

    # Build a Graph that computes predictions from the inference model.
    logits = mnist.inference(images,
                             FLAGS.hidden1,
                             FLAGS.hidden2)

    # Add to the Graph the loss calculation.
    loss = mnist.loss(logits, labels)

    # Add to the Graph operations that train the model.
    train_op = mnist.training(loss, FLAGS.learning_rate)

    # The op for initializing the variables.
    init_op = tf.group(tf.global_variables_initializer(),
                       tf.local_variables_initializer())

    # Create a session for running operations in the Graph.
    sess = tf.Session()

    # Initialize the variables (the trained variables and the
    # epoch counter).
    sess.run(init_op)

    # Start input enqueue threads.
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)

    try:
      step = 0
      while not coord.should_stop():
        start_time = time.time()

        # Run one step of the model.  The return values are
        # the activations from the `train_op` (which is
        # discarded) and the `loss` op.  To inspect the values
        # of your ops or variables, you may include them in
        # the list passed to sess.run() and the value tensors
        # will be returned in the tuple from the call.
        _, loss_value = sess.run([train_op, loss])

        duration = time.time() - start_time

        # Print an overview fairly often.
        if step % 100 == 0:
          print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value,
                                                     duration))
        step += 1
    except tf.errors.OutOfRangeError:
      print('Done training for %d epochs, %d steps.' % (FLAGS.num_epochs, step))
    finally:
      # When done, ask the threads to stop.
      coord.request_stop()

    # Wait for threads to finish.
    coord.join(threads)
    sess.close()
//  [python]
def create_or_load_model(model, model_dir, session, name):
  """Create translation model and initialize or load parameters in session."""
  latest_ckpt = tf.train.latest_checkpoint(model_dir)
  if latest_ckpt:
    model = load_model(model, latest_ckpt, session, name)
  else:
    start_time = time.time()
    session.run(tf.initialize_all_variables())
    session.run(tf.tables_initializer())
    utils.print_out("  created %s model with fresh parameters, time %.2fs" %
                    (name, time.time() - start_time))

  global_step = model.global_step.eval(session=session)
  return model, global_step
//  [python]
def create_or_load_model(model, model_dir, session, name):
  """Create translation model and initialize or load parameters in session."""
  latest_ckpt = tf.train.latest_checkpoint(model_dir)
  if latest_ckpt:
    model = load_model(model, latest_ckpt, session, name)
  else:
    start_time = time.time()
    session.run(tf.global_variables_initializer())
    session.run(tf.tables_initializer())
    utils.print_out("  created %s model with fresh parameters, time %.2fs" %
                    (name, time.time() - start_time))

  global_step = model.global_step.eval(session=session)
  return model, global_step
//  [python]
def main(_):
  if not FLAGS.pos_data_path:
    raise ValueError("No data files found in 'data_path' folder")

  raw_data = reader.load_data(FLAGS.pos_data_path)
  train_word, train_tag, dev_word, dev_tag, test_word, test_tag, vocabulary = raw_data
  
  config = get_config(FLAGS.pos_lang)
  eval_config = get_config(FLAGS.pos_lang)
  eval_config.batch_size = 1
  eval_config.num_steps = 1
  
  with tf.Graph().as_default(), tf.Session() as session:
    initializer = tf.random_uniform_initializer(-config.init_scale,
                                                config.init_scale)
    with tf.variable_scope(FLAGS.pos_scope_name, reuse=None, initializer=initializer):
      m = POSTagger(is_training=True, config=config)
    with tf.variable_scope(FLAGS.pos_scope_name, reuse=True, initializer=initializer):
      mvalid = POSTagger(is_training=False, config=config)
      mtest = POSTagger(is_training=False, config=eval_config)
    
    # CheckPoint State
    ckpt = tf.train.get_checkpoint_state(FLAGS.pos_train_dir)
    if ckpt:
      print("Loading model parameters from %s" % ckpt.model_checkpoint_path)
      m.saver.restore(session, tf.train.latest_checkpoint(FLAGS.pos_train_dir))
    else:
      print("Created model with fresh parameters.")
      session.run(tf.initialize_all_variables())
    
    for i in range(config.max_max_epoch):
      lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
      m.assign_lr(session, config.learning_rate * lr_decay)
        
      print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
      train_perplexity = run_epoch(session, m, train_word, train_tag, m.train_op,
                                   verbose=True)
      print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
      valid_perplexity = run_epoch(session, mvalid, dev_word, dev_tag, tf.no_op())
      print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
    
    test_perplexity = run_epoch(session, mtest, test_word,test_tag, tf.no_op())
    print("Test Perplexity: %.3f" % test_perplexity)
//  [python]
def main(_):
  if not FLAGS.pos_data_path:
    raise ValueError("No data files found in 'data_path' folder")

  raw_data = reader.load_data(FLAGS.pos_data_path)
  train_word, train_tag, dev_word, dev_tag, test_word, test_tag, vocabulary = raw_data
  
  config = get_config(FLAGS.pos_lang)
  eval_config = get_config(FLAGS.pos_lang)
  eval_config.batch_size = 1
  eval_config.num_steps = 1
  
  with tf.Graph().as_default(), tf.Session() as session:
    initializer = tf.random_uniform_initializer(-config.init_scale,
                                                config.init_scale)
    with tf.variable_scope(FLAGS.pos_scope_name, reuse=None, initializer=initializer):
      m = POSTagger(is_training=True, config=config)
    with tf.variable_scope(FLAGS.pos_scope_name, reuse=True, initializer=initializer):
      mvalid = POSTagger(is_training=False, config=config)
      mtest = POSTagger(is_training=False, config=eval_config)
    
    # CheckPoint State
    ckpt = tf.train.get_checkpoint_state(FLAGS.pos_train_dir)
    if ckpt:
      print("Loading model parameters from %s" % ckpt.model_checkpoint_path)
      m.saver.restore(session, tf.train.latest_checkpoint(FLAGS.pos_train_dir))
    else:
      print("Created model with fresh parameters.")
      session.run(tf.global_variables_initializer())
    
    for i in range(config.max_max_epoch):
      lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
      m.assign_lr(session, config.learning_rate * lr_decay)
        
      print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
      train_perplexity = run_epoch(session, m, train_word, train_tag, m.train_op,
                                   verbose=True)
      print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
      valid_perplexity = run_epoch(session, mvalid, dev_word, dev_tag, tf.no_op())
      print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
    
    test_perplexity = run_epoch(session, mtest, test_word,test_tag, tf.no_op())
    print("Test Perplexity: %.3f" % test_perplexity)
//  [python]
def test2():
    from Dataset import Dataset
    import matplotlib.pyplot as plt
    import numpy as np

    dataset = Dataset(datadir="/media/data/marc/tfrecords/fields/L1C/240", verbose=True, temporal_samples=30,section='2016')

    training_dataset, output_shapes, output_datatypes, fm_train = dataset.create_tf_dataset("eval", fold=0, batchsize=2, shuffle=False, prefetch_batches=1, num_batches=-1, threads=8, drop_remainder=False)

    iterator = training_dataset.make_initializable_iterator()

    model = Model()
    model.build_graph(modelpath=FLAGS.modelfolder)

    def get_operation(name):
        return tf.get_default_graph().get_operation_by_name(name).outputs[0]

    iterator_handle_op = get_operation("data_iterator_handle")
    is_train_op = get_operation("is_train")

    mpp_op = get_operation("mask_per_px")
    lpp_op = get_operation("loss_per_px")
    #umlpp_op = get_operation("unmasked_loss_per_px")

    with tf.Session() as sess:
        sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])

        sess.run([iterator.initializer, tf.tables_initializer()])


        data_handle = sess.run(iterator.string_handle())

        feed_dict = {iterator_handle_op: data_handle, is_train_op: False}

        y,labels,alllabels,mpp,lpp = sess.run([model.y,model.labels,model.alllabels,mpp_op,lpp_op],feed_dict=feed_dict)
        #x10, x20, x60, doy, year, labels = sess.run(iterator.get_next())
        a = sess.run(tf.one_hot(alllabels-1, model.num_classes))

        plt.imshow(y[0,0])
//  [python]
def dump_tfhub_to_hdf5(module_path, hdf5_path, redownload=False):
  """Loads TFHub weights and saves them to intermediate HDF5 file.

  Args:
    module_path ([Path-like]): Path to TFHub module.
    hdf5_path ([Path-like]): Path to output HDF5 file.

  Returns:
    [h5py.File]: Loaded hdf5 file containing module weights.
  """
  if os.path.exists(hdf5_path) and (not redownload):
    print('Loading BigGAN hdf5 file from:', hdf5_path)
    return h5py.File(hdf5_path, 'r')

  print('Loading BigGAN module from:', module_path)
  tf.reset_default_graph()
  hub.Module(module_path)
  print('Loaded BigGAN module from:', module_path)

  initializer = tf.initialize_all_variables()
  sess = tf.Session()
  sess.run(initializer)

  print('Saving BigGAN weights to :', hdf5_path)
  h5f = h5py.File(hdf5_path, 'w')
  for var in tf.global_variables():
    val = sess.run(var)
    h5f.create_dataset(var.name, data=val)
    print(f'Saving {var.name} with shape {val.shape}')
  h5f.close()
  return h5py.File(hdf5_path, 'r')
//  [python]
def dump_tfhub_to_hdf5(module_path, hdf5_path, redownload=False):
  """Loads TFHub weights and saves them to intermediate HDF5 file.

  Args:
    module_path ([Path-like]): Path to TFHub module.
    hdf5_path ([Path-like]): Path to output HDF5 file.

  Returns:
    [h5py.File]: Loaded hdf5 file containing module weights.
  """
  if os.path.exists(hdf5_path) and (not redownload):
    print('Loading BigGAN hdf5 file from:', hdf5_path)
    return h5py.File(hdf5_path, 'r')

  print('Loading BigGAN module from:', module_path)
  tf.reset_default_graph()
  hub.Module(module_path)
  print('Loaded BigGAN module from:', module_path)

  initializer = tf.global_variables_initializer()
  sess = tf.Session()
  sess.run(initializer)

  print('Saving BigGAN weights to :', hdf5_path)
  h5f = h5py.File(hdf5_path, 'w')
  for var in tf.global_variables():
    val = sess.run(var)
    h5f.create_dataset(var.name, data=val)
    print(f'Saving {var.name} with shape {val.shape}')
  h5f.close()
  return h5py.File(hdf5_path, 'r')
//  [python]
def _save_tf_model(self):
        ckpt_dir = "/opt/ml/output/data/checkpoint"
        model_dir = "/opt/ml/model"

        import tensorflow as tf  # importing tensorflow here so that MXNet docker image is compatible with this file.

        # Re-Initialize from the checkpoint so that you will have the latest models up.
        tf.train.init_from_checkpoint(
            ckpt_dir, {"main_level/agent/online/network_0/": "main_level/agent/online/network_0"}
        )
        tf.train.init_from_checkpoint(
            ckpt_dir, {"main_level/agent/online/network_1/": "main_level/agent/online/network_1"}
        )

        # Create a new session with a new tf graph.
        sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
        sess.run(tf.global_variables_initializer())  # initialize the checkpoint.

        # This is the node that will accept the input.
        input_nodes = tf.get_default_graph().get_tensor_by_name(
            "main_level/agent/main/online/" + "network_0/observation/observation:0"
        )
        # This is the node that will produce the output.
        output_nodes = tf.get_default_graph().get_operation_by_name(
            "main_level/agent/main/online/" + "network_1/ppo_head_0/policy"
        )
        # Save the model as a servable model.
        tf.saved_model.simple_save(
            session=sess,
            export_dir="model",
            inputs={"observation": input_nodes},
            outputs={"policy": output_nodes.outputs[0]},
        )
        # Move to the appropriate folder. Don't mind the directory, this just works.
        # rl-cart-pole is the name of the model. Remember it.
        shutil.move("model/", model_dir + "/model/tf-model/00000001/")
        # EASE will pick it up and upload to the right path.
        print("Success")
//  [python]
def main(_):
  if not FLAGS.pos_data_path:
    raise ValueError("No data files found in 'data_path' folder")
  # Load Data
  raw_data = reader.load_data(FLAGS.pos_data_path)
  train_word, train_tag, dev_word, dev_tag, test_word, test_tag, vocab_size = raw_data
  
  # Load Config
  config_dict = load_config(FLAGS.pos_model_config_path)
  config = get_config(config_dict, FLAGS.pos_lang)
  eval_config = get_config(config_dict, FLAGS.pos_lang)
  eval_config.batch_size = 1
  eval_config.num_steps = 1
  
  # Define Variable Scope
  model_var_scope = get_model_var_scope(FLAGS.pos_scope_name, FLAGS.pos_lang)

  with tf.Graph().as_default(), tf.Session() as session:
    initializer = tf.random_uniform_initializer(-config.init_scale,
                                                config.init_scale)
    with tf.variable_scope(model_var_scope, reuse=False, initializer=initializer):
      m = POSTagger(is_training=True, config=config)
    with tf.variable_scope(model_var_scope, reuse=True, initializer=initializer):
      mvalid = POSTagger(is_training=False, config=config)
      mtest = POSTagger(is_training=False, config=eval_config)
    
    # CheckPoint State
    ckpt = tf.train.get_checkpoint_state(FLAGS.pos_train_dir)
    if ckpt:
      print("Loading model parameters from %s" % ckpt.model_checkpoint_path)
      m.saver.restore(session, tf.train.latest_checkpoint(FLAGS.pos_train_dir))
    else:
      print("Created model with fresh parameters.")
      session.run(tf.initialize_all_variables())
    
    # write the graph out for further use e.g. C++ API call
    tf.train.write_graph(session.graph_def, './models/', 'pos_graph.pbtxt', as_text=True)   # output is text
    
    for i in range(config.max_max_epoch):
      lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
      m.assign_lr(session, config.learning_rate * lr_decay)

      print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
      train_perplexity = run_epoch(session, m, train_word, train_tag, m.train_op,
                                   verbose=True)
      print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
      valid_perplexity = run_epoch(session, mvalid, dev_word, dev_tag, tf.no_op())
      print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
    
    test_perplexity = run_epoch(session, mtest, test_word, test_tag, tf.no_op())
    print("Test Perplexity: %.3f" % test_perplexity)
//  [python]
def main(_):
  if not FLAGS.pos_data_path:
    raise ValueError("No data files found in 'data_path' folder")
  # Load Data
  raw_data = reader.load_data(FLAGS.pos_data_path)
  train_word, train_tag, dev_word, dev_tag, test_word, test_tag, vocab_size = raw_data
  
  # Load Config
  config_dict = load_config(FLAGS.pos_model_config_path)
  config = get_config(config_dict, FLAGS.pos_lang)
  eval_config = get_config(config_dict, FLAGS.pos_lang)
  eval_config.batch_size = 1
  eval_config.num_steps = 1
  
  # Define Variable Scope
  model_var_scope = get_model_var_scope(FLAGS.pos_scope_name, FLAGS.pos_lang)

  with tf.Graph().as_default(), tf.Session() as session:
    initializer = tf.random_uniform_initializer(-config.init_scale,
                                                config.init_scale)
    with tf.variable_scope(model_var_scope, reuse=False, initializer=initializer):
      m = POSTagger(is_training=True, config=config)
    with tf.variable_scope(model_var_scope, reuse=True, initializer=initializer):
      mvalid = POSTagger(is_training=False, config=config)
      mtest = POSTagger(is_training=False, config=eval_config)
    
    # CheckPoint State
    ckpt = tf.train.get_checkpoint_state(FLAGS.pos_train_dir)
    if ckpt:
      print("Loading model parameters from %s" % ckpt.model_checkpoint_path)
      m.saver.restore(session, tf.train.latest_checkpoint(FLAGS.pos_train_dir))
    else:
      print("Created model with fresh parameters.")
      session.run(tf.global_variables_initializer())
    
    # write the graph out for further use e.g. C++ API call
    tf.train.write_graph(session.graph_def, './models/', 'pos_graph.pbtxt', as_text=True)   # output is text
    
    for i in range(config.max_max_epoch):
      lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
      m.assign_lr(session, config.learning_rate * lr_decay)

      print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
      train_perplexity = run_epoch(session, m, train_word, train_tag, m.train_op,
                                   verbose=True)
      print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
      valid_perplexity = run_epoch(session, mvalid, dev_word, dev_tag, tf.no_op())
      print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
    
    test_perplexity = run_epoch(session, mtest, test_word, test_tag, tf.no_op())
    print("Test Perplexity: %.3f" % test_perplexity)
//  [python]
def _run_and_test(self, hparams):
        # Construct database
        scalar_data = tx.data.ScalarData(hparams)

        self.assertEqual(scalar_data.list_items()[0],
                         hparams["dataset"]["data_name"])

        iterator = scalar_data.dataset.make_initializable_iterator()
        data_batch = iterator.get_next()

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())
            sess.run(iterator.initializer)

            i = 0
            while True:
                try:
                    # Run the logics
                    data_batch_ = sess.run(data_batch)
                    self.assertEqual(set(data_batch_.keys()),
                                     set(scalar_data.list_items()))
                    value = data_batch_[scalar_data.data_name][0]
                    self.assertEqual(i, value)
                    i += 1
                    # pylint: disable=no-member
                    if hparams["dataset"]["data_type"] == "int":
                        self.assertTrue(isinstance(value, np.int32))
                    else:
                        self.assertTrue(isinstance(value, np.float32))
                except tf.errors.OutOfRangeError:
                    print('Done -- epoch limit reached')
                    break
//  [python]
def test_shuffle(self):
        """Tests results of toggling shuffle.
        """
        hparams = copy.copy(self._int_hparams)
        hparams["batch_size"] = 10
        scalar_data = tx.data.ScalarData(hparams)
        iterator = scalar_data.dataset.make_initializable_iterator()
        data_batch = iterator.get_next()

        hparams_sfl = copy.copy(hparams)
        hparams_sfl["shuffle"] = True
        scalar_data_sfl = tx.data.ScalarData(hparams_sfl)
        iterator_sfl = scalar_data_sfl.dataset.make_initializable_iterator()
        data_batch_sfl = iterator_sfl.get_next()

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            sess.run(tf.tables_initializer())
            sess.run(iterator.initializer)
            sess.run(iterator_sfl.initializer)

            vals = []
            vals_sfl = []
            while True:
                try:
                    # Run the logics
                    data_batch_, data_batch_sfl_ = sess.run([data_batch,
                                                             data_batch_sfl])
                    vals += data_batch_[scalar_data.data_name].tolist()
                    vals_sfl += data_batch_sfl_[scalar_data.data_name].tolist()
                except tf.errors.OutOfRangeError:
                    print('Done -- epoch limit reached')
                    break
            self.assertEqual(len(vals), len(vals_sfl))
            self.assertSetEqual(set(vals), set(vals_sfl))
//  [python]
def train():
    # load parameters
    hparams = create_hparams()
    start_time = time.time()
    print("preparing train and dev data")
    # load dict message
    [hparams.word2id, hparams.pos2id, hparams.role2id, hparams.id2word, hparams.id2pos, hparams.id2role] = data_helper.load_dict(hparams=hparams)
    # [word, pos] ==> role
    Train_word, Train_pos, Train_role, Dev_word, Dev_pos, Dev_role = data_helper.get_train(hparams=hparams)
    
    print("building model...")
    with tf.Graph().as_default():
        config = tf.ConfigProto(allow_soft_placement=True)
        os.environ["CUDA_VISIBLE_DEVICES"] = str(hparams.gpu)
        sess = tf.Session(config=config)
        with sess.as_default():
            with tf.device("/gpu:" + str(hparams.gpu)):
                initializer = tf.random_uniform_initializer(-0.1, 0.1)
                with tf.variable_scope("model", reuse=None, initializer=initializer):
                    model = BILSTM_CRF.bilstm_crf(hparams=hparams)
                print("training model...")
                sess.run(tf.initialize_all_variables())
                model.train(sess, hparams, Train_word, Train_pos, Train_role, Dev_word, Dev_pos, Dev_role)
                print("final best f1 on valid dataset is: %f" % hparams.max_f1)

    end_time = time.time()
    print("time used %f (hour)" % ((end_time - start_time) / 3600))
    return
//  [python]
def train():
    # load parameters
    hparams = create_hparams()
    start_time = time.time()
    print("preparing train and dev data")
    # load dict message
    [hparams.word2id, hparams.pos2id, hparams.role2id, hparams.id2word, hparams.id2pos, hparams.id2role] = data_helper.load_dict(hparams=hparams)
    # [word, pos] ==> role
    Train_word, Train_pos, Train_role, Dev_word, Dev_pos, Dev_role = data_helper.get_train(hparams=hparams)
    
    print("building model...")
    with tf.Graph().as_default():
        config = tf.ConfigProto(allow_soft_placement=True)
        os.environ["CUDA_VISIBLE_DEVICES"] = str(hparams.gpu)
        sess = tf.Session(config=config)
        with sess.as_default():
            with tf.device("/gpu:" + str(hparams.gpu)):
                initializer = tf.random_uniform_initializer(-0.1, 0.1)
                with tf.variable_scope("model", reuse=None, initializer=initializer):
                    model = BILSTM_CRF.bilstm_crf(hparams=hparams)
                print("training model...")
                sess.run(tf.global_variables_initializer())
                model.train(sess, hparams, Train_word, Train_pos, Train_role, Dev_word, Dev_pos, Dev_role)
                print("final best f1 on valid dataset is: %f" % hparams.max_f1)

    end_time = time.time()
    print("time used %f (hour)" % ((end_time - start_time) / 3600))
    return
//  [python]
def sample_policy(save_dir=None, savescope='policy', temperature=1.0, seed=None, batch_size=4, nsamples=0):
    hparams = train_policy.HParams()
    hparams.override_from_json_file(os.path.join(save_dir, 'train_policy_hparams.json'))
    print('hparams', hparams)
    task = hparams.task

    comm = MPI.COMM_WORLD
    nsamples_per_rank = utils.exact_div(nsamples, comm.Get_size())
    with tf.Graph().as_default():
        m = trained_models.TrainedModel(name='sample', savedir=os.path.join(save_dir, 'policy'), scope='policy')
        encoder = m.encoding.get_encoder()
        hyperparams.dump(m.hparams(), name='model_hparams')

        utils.set_mpi_seed(seed)

        policy = Policy(
            m, scope='policy',
            is_root=True, # just init on every rank, simplifies code
            embed_queries=lm_tasks.query_formatter(task, encoder),
            temperature=temperature,
        )

        query_sampler = lm_tasks.make_query_sampler(
            hparams=task, encoder=encoder, comm=comm,
            batch_size=batch_size, mode='test'
        )

        init_ops = tf.group(
            tf.global_variables_initializer(),
            tf.local_variables_initializer(),
        )

        with utils.mpi_session() as sess:
            init_ops.run()
            @utils.graph_function()
            def sample_queries():
                return query_sampler()['tokens']

            tf.get_default_graph().finalize()

            generated = 0
            while nsamples_per_rank == 0 or generated < nsamples_per_rank:
                queries = sample_queries()
                rollouts = policy.respond(queries, length=task.response_length)
                assert len(queries.tolist()) == batch_size
                assert len(rollouts['responses'].tolist()) == batch_size
                for q, r in zip(queries.tolist(), rollouts['responses'].tolist()):
                    print('=' * 80)
                    print(encoder.decode(q).replace("\n", "⏎"))
                    print(encoder.decode(r).replace("\n", "⏎"))
                generated += batch_size
//  [python]
def testEvaluation(self):
    batch_size = 2
    height, width = 299, 299
    num_classes = 1000
    with self.test_session() as sess:
      eval_inputs = tf.random_uniform((batch_size, height, width, 3))
      logits, _ = inception.inception_v4(eval_inputs,
                                         num_classes,
                                         is_training=False)
      predictions = tf.argmax(logits, 1)
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def testEvaluation(self):
    batch_size = 2
    height, width = 299, 299
    num_classes = 1000
    with self.test_session() as sess:
      eval_inputs = tf.random_uniform((batch_size, height, width, 3))
      logits, _ = inception.inception_v4(eval_inputs,
                                         num_classes,
                                         is_training=False)
      predictions = tf.argmax(logits, 1)
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def testUnknownBatchSize(self):
    batch_size = 1
    height, width = 299, 299
    num_classes = 1000
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, (None, height, width, 3))
      logits, _ = inception.inception_v4(inputs, num_classes)
      self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [None, num_classes])
      images = tf.random_uniform((batch_size, height, width, 3))
      sess.run(tf.initialize_all_variables())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch_size, num_classes))
//  [python]
def testUnknownBatchSize(self):
    batch_size = 1
    height, width = 299, 299
    num_classes = 1000
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, (None, height, width, 3))
      logits, _ = inception.inception_v4(inputs, num_classes)
      self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [None, num_classes])
      images = tf.random_uniform((batch_size, height, width, 3))
      sess.run(tf.global_variables_initializer())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch_size, num_classes))
//  [python]
def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000
    with self.test_session() as sess:
      train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
      inception.inception_v4(train_inputs, num_classes)
      eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
      logits, _ = inception.inception_v4(eval_inputs,
                                         num_classes,
                                         is_training=False,
                                         reuse=True)
      predictions = tf.argmax(logits, 1)
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,))
//  [python]
def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000
    with self.test_session() as sess:
      train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
      inception.inception_v4(train_inputs, num_classes)
      eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
      logits, _ = inception.inception_v4(eval_inputs,
                                         num_classes,
                                         is_training=False,
                                         reuse=True)
      predictions = tf.argmax(logits, 1)
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,))
//  [python]
def testGlobalPoolUnknownImageShape(self):
    batch_size = 1
    height, width = 350, 400
    num_classes = 1000
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, (batch_size, None, None, 3))
      logits, end_points = inception.inception_v4(
          inputs, num_classes, create_aux_logits=False)
      self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Mixed_7d']
      images = tf.random_uniform((batch_size, height, width, 3))
      sess.run(tf.initialize_all_variables())
      logits_out, pre_pool_out = sess.run([logits, pre_pool],
                                          {inputs: images.eval()})
      self.assertTupleEqual(logits_out.shape, (batch_size, num_classes))
      self.assertTupleEqual(pre_pool_out.shape, (batch_size, 9, 11, 1536))
//  [python]
def testGlobalPoolUnknownImageShape(self):
    batch_size = 1
    height, width = 350, 400
    num_classes = 1000
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, (batch_size, None, None, 3))
      logits, end_points = inception.inception_v4(
          inputs, num_classes, create_aux_logits=False)
      self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Mixed_7d']
      images = tf.random_uniform((batch_size, height, width, 3))
      sess.run(tf.global_variables_initializer())
      logits_out, pre_pool_out = sess.run([logits, pre_pool],
                                          {inputs: images.eval()})
      self.assertTupleEqual(logits_out.shape, (batch_size, num_classes))
      self.assertTupleEqual(pre_pool_out.shape, (batch_size, 9, 11, 1536))
//  [python]
def test_nonfreeze():
    np.random.seed(0)
    tf.set_random_seed(0)

    a = tf.Variable(np.random.randn(3).astype('float32'))
    b = tf.Variable(np.random.randn(2,5).astype('float32'))
    loss = tf.reduce_sum(tf.square(a)) + tf.reduce_sum(tf.sin(b))

    stepsize = 1e-2
    # for some reason the session config with inter_op_parallelism_threads was causing
    # nested sess.run calls to freeze
    config = tf.ConfigProto(inter_op_parallelism_threads=1)
    sess = U.get_session(config=config)
    update_op = MpiAdamOptimizer(comm=MPI.COMM_WORLD, learning_rate=stepsize).minimize(loss)
    sess.run(tf.initialize_all_variables())
    losslist_ref = []
    for i in range(100):
        l,_ = sess.run([loss, update_op])
        print(i, l)
        losslist_ref.append(l)
//  [python]
def test_nonfreeze():
    np.random.seed(0)
    tf.set_random_seed(0)

    a = tf.Variable(np.random.randn(3).astype('float32'))
    b = tf.Variable(np.random.randn(2,5).astype('float32'))
    loss = tf.reduce_sum(tf.square(a)) + tf.reduce_sum(tf.sin(b))

    stepsize = 1e-2
    # for some reason the session config with inter_op_parallelism_threads was causing
    # nested sess.run calls to freeze
    config = tf.ConfigProto(inter_op_parallelism_threads=1)
    sess = U.get_session(config=config)
    update_op = MpiAdamOptimizer(comm=MPI.COMM_WORLD, learning_rate=stepsize).minimize(loss)
    sess.run(tf.global_variables_initializer())
    losslist_ref = []
    for i in range(100):
        l,_ = sess.run([loss, update_op])
        print(i, l)
        losslist_ref.append(l)
//  [python]
def testBatchNorm(self):
      
        tf.set_random_seed(123)
  
        x = tf.placeholder(tf.float32, [None, 20, 20, 10], name='input')
        phase_train = tf.placeholder(tf.bool, name='phase_train')
        
        # generate random noise to pass into batch norm
        #x_gen = tf.random_normal([50,20,20,10])
        
        bn = models.network.batch_norm(x, phase_train)
        
        init = tf.initialize_all_variables()
        sess = tf.Session(config=tf.ConfigProto())
        sess.run(init)
  
        with sess.as_default():
        
            #generate a constant variable to pass into batch norm
            y = np.random.normal(0, 1, size=(50,20,20,10))
            
            feed_dict = {x: y, phase_train: True}
            sess.run(bn, feed_dict=feed_dict)
            
            feed_dict = {x: y, phase_train: False}
            y1 = sess.run(bn, feed_dict=feed_dict)
            y2 = sess.run(bn, feed_dict=feed_dict)
            
            testing.assert_almost_equal(y1, y2, 10, 'Output from two forward passes with phase_train==false should be equal')
//  [python]
def testBatchNorm(self):
      
        tf.set_random_seed(123)
  
        x = tf.placeholder(tf.float32, [None, 20, 20, 10], name='input')
        phase_train = tf.placeholder(tf.bool, name='phase_train')
        
        # generate random noise to pass into batch norm
        #x_gen = tf.random_normal([50,20,20,10])
        
        bn = models.network.batch_norm(x, phase_train)
        
        init = tf.global_variables_initializer()
        sess = tf.Session(config=tf.ConfigProto())
        sess.run(init)
  
        with sess.as_default():
        
            #generate a constant variable to pass into batch norm
            y = np.random.normal(0, 1, size=(50,20,20,10))
            
            feed_dict = {x: y, phase_train: True}
            sess.run(bn, feed_dict=feed_dict)
            
            feed_dict = {x: y, phase_train: False}
            y1 = sess.run(bn, feed_dict=feed_dict)
            y2 = sess.run(bn, feed_dict=feed_dict)
            
            testing.assert_almost_equal(y1, y2, 10, 'Output from two forward passes with phase_train==false should be equal')
//  [python]
def initialize_graph(self):
        with self.sess.graph.as_default():
            features, labels = get_inference_input()
            params = FLAGS.flag_values_dict()
            logging.info('TPU inference is supported on C++ only. '
                         'DualNetwork will ignore use_tpu=True')
            params['use_tpu'] = False
            estimator_spec = model_fn(features, labels,
                                      tf.estimator.ModeKeys.PREDICT,
                                      params=params)
            self.inference_input = features
            self.inference_output = estimator_spec.predictions
            if self.save_file is not None:
                self.initialize_weights(self.save_file)
            else:
                self.sess.run(tf.global_variables_initializer())
//  [python]
def rollback(var_list, ckpt_folder, ckpt_file=None):
    """ This function provides a shortcut for reloading a model and calculating a list of variables

    :param var_list:
    :param ckpt_folder:
    :param ckpt_file: in case an older ckpt file is needed, provide it here, e.g. 'cifar.ckpt-6284'
    :return:
    """
    global_step = global_step_config()
    # register a session
    sess = tf.Session(config=tf.ConfigProto(
        allow_soft_placement=True,
        log_device_placement=False))
    # initialization
    init_op = tf.group(tf.initialize_all_variables(), tf.local_variables_initializer())
    sess.run(init_op)
    # load the training graph
    saver = tf.train.Saver(max_to_keep=2)
    ckpt = get_ckpt(ckpt_folder, ckpt_file=ckpt_file)
    if ckpt is None:
        raise FileNotFoundError('No ckpt Model found at {}.'.format(ckpt_folder))
    saver.restore(sess, ckpt.model_checkpoint_path)
    FLAGS.print('Model reloaded.')
    # run the session
    coord = tf.train.Coordinator()
    # threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    var_value, global_step_value = sess.run([var_list, global_step])
    coord.request_stop()
    # coord.join(threads)
    sess.close()
    FLAGS.print('Variable calculated.')

    return var_value, global_step_value
//  [python]
def rollback(var_list, ckpt_folder, ckpt_file=None):
    """ This function provides a shortcut for reloading a model and calculating a list of variables

    :param var_list:
    :param ckpt_folder:
    :param ckpt_file: in case an older ckpt file is needed, provide it here, e.g. 'cifar.ckpt-6284'
    :return:
    """
    global_step = global_step_config()
    # register a session
    sess = tf.Session(config=tf.ConfigProto(
        allow_soft_placement=True,
        log_device_placement=False))
    # initialization
    init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
    sess.run(init_op)
    # load the training graph
    saver = tf.train.Saver(max_to_keep=2)
    ckpt = get_ckpt(ckpt_folder, ckpt_file=ckpt_file)
    if ckpt is None:
        raise FileNotFoundError('No ckpt Model found at {}.'.format(ckpt_folder))
    saver.restore(sess, ckpt.model_checkpoint_path)
    FLAGS.print('Model reloaded.')
    # run the session
    coord = tf.train.Coordinator()
    # threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    var_value, global_step_value = sess.run([var_list, global_step])
    coord.request_stop()
    # coord.join(threads)
    sess.close()
    FLAGS.print('Variable calculated.')

    return var_value, global_step_value
//  [python]
def bootstrap():
    """Initialize a tf.Estimator run with random initial weights."""
    # a bit hacky - forge an initial checkpoint with the name that subsequent
    # Estimator runs will expect to find.
    #
    # Estimator will do this automatically when you call train(), but calling
    # train() requires data, and I didn't feel like creating training data in
    # order to run the full train pipeline for 1 step.
    maybe_set_seed()
    initial_checkpoint_name = 'model.ckpt-1'
    save_file = os.path.join(FLAGS.work_dir, initial_checkpoint_name)
    sess = tf.compat.v1.Session(graph=tf.Graph())
    with sess.graph.as_default():
        features, labels = get_inference_input()
        model_fn(features, labels, tf.estimator.ModeKeys.PREDICT,
                 params=FLAGS.flag_values_dict())
        sess.run(tf.compat.v1.global_variables_initializer())
        tf.compat.v1.train.Saver().save(sess, save_file)
//  [python]
def bootstrap():
    """Initialize a tf.Estimator run with random initial weights."""
    # a bit hacky - forge an initial checkpoint with the name that subsequent
    # Estimator runs will expect to find.
    #
    # Estimator will do this automatically when you call train(), but calling
    # train() requires data, and I didn't feel like creating training data in
    # order to run the full train pipeline for 1 step.
    maybe_set_seed()
    initial_checkpoint_name = 'model.ckpt-1'
    save_file = os.path.join(FLAGS.work_dir, initial_checkpoint_name)
    sess = tf.compat.v1.Session(graph=tf.Graph())
    with sess.graph.as_default():
        features, labels = get_inference_input()
        model_fn(features, labels, tf.estimator.ModeKeys.PREDICT,
                 params=FLAGS.flag_values_dict())
        sess.run(tf.global_variables_initializer())
        tf.compat.v1.train.Saver().save(sess, save_file)
//  [python]
def __init__(
            self, do_save=False, do_trace=False, save_path=None,
            load_ckpt=False, log_device=False, ckpt_var_list=None):
        """ This class provides shortcuts for running sessions.
        It needs to be run under context managers. Example:
        with MySession() as sess:
            var1_value, var2_value = sess.run_once([var1, var2])

        :param do_save:
        :param do_trace:
        :param save_path:
        :param load_ckpt:
        :param log_device:
        :param ckpt_var_list: list of variables to save / restore
        """
        # somehow it gives error: "global_step does not exist or is not created from tf.get_variable".
        # self.global_step = global_step_config()
        self.log_device = log_device
        # register a session
        # self.sess = tf.Session(config=tf.ConfigProto(
        #     allow_soft_placement=True,
        #     log_device_placement=log_device,
        #     gpu_options=tf.GPUOptions(allow_growth=True)))
        self.sess = tf.Session(config=tf.ConfigProto(
            allow_soft_placement=True,
            log_device_placement=log_device))
        # initialization
        init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
        self.sess.run(init_op)
        self.coord = None
        self.threads = None
        FLAGS.print('Graph initialization finished...')
        # configuration
        self.ckpt_var_list = ckpt_var_list
        if do_save:
            self.saver = self._get_saver_()
            self.save_path = save_path
        else:
            self.saver = None
            self.save_path = None
        self.summary_writer = None
        self.do_trace = do_trace
        self.load_ckpt = load_ckpt
//  [python]
def _make_model(self, graph, mode, batch_size):
        with graph.as_default():

            label_data = self._fetch_data(mode, batch_size)

            model = LanguageModel(
                label_data=label_data,
                mode=mode,
                hparams=self._hparams
            )

            global_vars_initializer = tf.global_variables_initializer()
            tables_initialiser = tf.tables_initializer()

            # Returning the original data, not the processed features
            return Model(data=label_data,
                         model=model,
                         initializer=[global_vars_initializer, tables_initialiser],
                         batch_size=batch_size)