jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def begin(self):
    bcast_vars = []
    embed_para_vars = ops.get_collection(constant.EmbeddingParallel)
    for x in tf.global_variables():
      # if '/embedding' not in x.name and 'DynamicVariable' not in str(type(x)):
      if x.name not in embed_para_vars:
        bcast_vars.append(x)
        logging.info('will broadcast variable: name=%s shape=%s' %
                     (x.name, x.get_shape()))
    if not self.bcast_op or self.bcast_op.graph != tf.get_default_graph():
      with tf.device(self.device):
        self.bcast_op = broadcast_variables(bcast_vars, self.root_rank)
//  [python]
def get_optimistic_restore_variables(ckpt_path, graph=tf.get_default_graph()):
    reader = tf.train.NewCheckpointReader(ckpt_path)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted(
        [
            (var.name, var.name.split(":")[0])
            for var in tf.global_variables()
            if var.name.split(":")[0] in saved_shapes
        ]
    )
    restore_vars = []
    for var_name, saved_var_name in var_names:
        var = graph.get_tensor_by_name(var_name)
        var_shape = var.get_shape().as_list()
        if var_shape == saved_shapes[saved_var_name]:
            restore_vars.append(var)
    return restore_vars
//  [python]
def freeze_session(sess, keep_var_names=None, output_names=None, clear_devices=True):
    """Freezes the state of a session into a pruned computation graph."""
    output_names = [i.replace(":0", "") for i in output_names]
    graph = sess.graph
    with graph.as_default():
        freeze_var_names = list(set(v.op.name for v in tf.global_variables()).difference(keep_var_names or []))
        output_names = output_names or []
        output_names += [v.op.name for v in tf.global_variables()]
        input_graph_def = graph.as_graph_def()
        if clear_devices:
            for node in input_graph_def.node:
                node.device = ""
        frozen_graph = convert_variables_to_constants(sess, input_graph_def,
                                                      output_names, freeze_var_names)
        return frozen_graph
//  [python]
def _checkpoint_var_search(self, checkpoint_path):
        reader = tf.train.NewCheckpointReader(checkpoint_path)
        saved_shapes = reader.get_variable_to_shape_map()
        model_names = tf.model_variables()  # Used by tf.slim layers
        if not len(tf.model_variables()):
            model_names = tf.global_variables()  # Fallback when slim is not used
        model_names = set([v.name.split(':')[0] for v in model_names])
        checkpoint_names = set(saved_shapes.keys())
        found_names = model_names & checkpoint_names
        missing_names = model_names - checkpoint_names
        shape_conflicts = set()
        restored = []
        with tf.variable_scope('', reuse=True):
            for name in found_names:
                var = tf.get_variable(name)
                var_shape = var.get_shape().as_list()
                if var_shape == saved_shapes[name]:
                    restored.append(var)
                else:
                    shape_conflicts.add(name)
        found_names -= shape_conflicts
        return (restored, sorted(found_names),
                sorted(missing_names), sorted(shape_conflicts))
//  [python]
def train(self, train, dev):
        """

        :param train:
        :param dev:
        :return:
        """
        saver = tf.train.Saver(tf.all_variables())

        with tf.Session(config=self.config) as sess:
            sess.run(self.init_op)
            self.add_summary(sess)

            for epoch in range(self.epoch_num):
                self.run_one_epoch(sess, train, dev, self.tag2label, epoch, saver)
//  [python]
def train(self, train, dev):
        """

        :param train:
        :param dev:
        :return:
        """
        saver = tf.train.Saver(tf.global_variables())

        with tf.Session(config=self.config) as sess:
            sess.run(self.init_op)
            self.add_summary(sess)

            for epoch in range(self.epoch_num):
                self.run_one_epoch(sess, train, dev, self.tag2label, epoch, saver)
//  [python]
def load(self, dir_name, epoch=0, name=None):
        """save model to dir

        Parameters
        ----------
        dir_name: str
            name of the directory
        epoch: int
        """
        if name is None or name == self.name:  # the name of saved model is the same as ours
            dir_name = os.path.join(dir_name, self.name)
            model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name)
            saver = tf.train.Saver(model_vars)
            saver.restore(self.sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch))
        else:  # load a checkpoint with different name
            backup_graph = tf.get_default_graph()
            kv_dict = {}

            # load checkpoint from another saved graph
            with tf.Graph().as_default(), tf.Session() as sess:
                tf.train.import_meta_graph(os.path.join(dir_name, name, (self.subclass_name + "_%d") % epoch + ".meta"))
                dir_name = os.path.join(dir_name, name)
                model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name)
                sess.run(tf.global_variables_initializer())
                saver = tf.train.Saver(model_vars)
                saver.restore(sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch))
                for item in tf.all_variables():
                    kv_dict[item.name] = sess.run(item)

            # assign to now graph
            backup_graph.as_default()
            model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name)
            for item in model_vars:
                old_name = item.name.replace(self.name, name)
                self.sess.run(tf.assign(item, kv_dict[old_name]))
//  [python]
def load(self, dir_name, epoch=0, name=None):
        """save model to dir

        Parameters
        ----------
        dir_name: str
            name of the directory
        epoch: int
        """
        if name is None or name == self.name:  # the name of saved model is the same as ours
            dir_name = os.path.join(dir_name, self.name)
            model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name)
            saver = tf.train.Saver(model_vars)
            saver.restore(self.sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch))
        else:  # load a checkpoint with different name
            backup_graph = tf.get_default_graph()
            kv_dict = {}

            # load checkpoint from another saved graph
            with tf.Graph().as_default(), tf.Session() as sess:
                tf.train.import_meta_graph(os.path.join(dir_name, name, (self.subclass_name + "_%d") % epoch + ".meta"))
                dir_name = os.path.join(dir_name, name)
                model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name)
                sess.run(tf.global_variables_initializer())
                saver = tf.train.Saver(model_vars)
                saver.restore(sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch))
                for item in tf.global_variables():
                    kv_dict[item.name] = sess.run(item)

            # assign to now graph
            backup_graph.as_default()
            model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name)
            for item in model_vars:
                old_name = item.name.replace(self.name, name)
                self.sess.run(tf.assign(item, kv_dict[old_name]))
//  [python]
def load_variables_from_checkpoint(sess, start_checkpoint):
  """Utility function to centralize checkpoint restoration.

  Args:
    sess: TensorFlow session.
    start_checkpoint: Path to saved checkpoint on disk.
  """
  saver = tf.train.Saver(tf.all_variables())
  saver.restore(sess, start_checkpoint)
//  [python]
def load_variables_from_checkpoint(sess, start_checkpoint):
  """Utility function to centralize checkpoint restoration.

  Args:
    sess: TensorFlow session.
    start_checkpoint: Path to saved checkpoint on disk.
  """
  saver = tf.train.Saver(tf.global_variables())
  saver.restore(sess, start_checkpoint)
//  [python]
def save_model_params(sess,
                      file_path):
    # assert file_path.endswith('.npz')
    param_dict = {v.name: v.eval(sess) for v in tf.all_variables()}
    np.savez_compressed(file_path, **param_dict)
//  [python]
def save_model_params(sess,
                      file_path):
    # assert file_path.endswith('.npz')
    param_dict = {v.name: v.eval(sess) for v in tf.global_variables()}
    np.savez_compressed(file_path, **param_dict)
//  [python]
def restore_map(self, fine_tune_checkpoint_type='detection'):
    """Returns a map of variables to load from a foreign checkpoint.

    Args:
      fine_tune_checkpoint_type: whether to restore from a full detection
        checkpoint (with compatible variable names) or to restore from a
        classification checkpoint for initialization prior to training.
        Valid values: `detection`, `classification`. Default 'detection'.

    Returns:
      A dict mapping variable names to variables.
    """
    return {var.op.name: var for var in tf.global_variables()}
//  [python]
def evaluate(self):
        config = tf.ConfigProto(allow_soft_placement=True)
        ###############
        custom_op = config.graph_options.rewrite_options.custom_optimizers.add()
        custom_op.name = "NpuOptimizer"
        custom_op.parameter_map["use_off_line"].b = True
        custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision")
        config.graph_options.rewrite_options.remapping = RewriterConfig.OFF
        config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF
        
        config.gpu_options.per_process_gpu_memory_fraction = 0.7
        with tf.Session(config=config) as sess:
            with tf.device("/gpu:%d" % cfg.GPU_ID):
            # with 1:
                if self.model_path.find('.ckpt') != -1:
                    self.init_opt()
                    print("Reading model parameters from %s" % self.model_path)
                    saver = tf.train.Saver(tf.all_variables())
                    print(tf.all_variables())
                    saver.restore(sess, self.model_path)

                    # self.eval_one_dataset(sess, self.dataset.train, self.log_dir, subset='train')

                    self.eval_one_dataset(sess, self.dataset.test, self.log_dir, subset='test')
                else:
                    print("Input a valid model path.")
//  [python]
def evaluate(self):
        config = tf.ConfigProto(allow_soft_placement=True)
        ###############
        custom_op = config.graph_options.rewrite_options.custom_optimizers.add()
        custom_op.name = "NpuOptimizer"
        custom_op.parameter_map["use_off_line"].b = True
        custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision")
        config.graph_options.rewrite_options.remapping = RewriterConfig.OFF
        config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF
        
        config.gpu_options.per_process_gpu_memory_fraction = 0.7
        with tf.Session(config=config) as sess:
            with tf.device("/gpu:%d" % cfg.GPU_ID):
            # with 1:
                if self.model_path.find('.ckpt') != -1:
                    self.init_opt()
                    print("Reading model parameters from %s" % self.model_path)
                    saver = tf.train.Saver(tf.global_variables())
                    print(tf.global_variables())
                    saver.restore(sess, self.model_path)

                    # self.eval_one_dataset(sess, self.dataset.train, self.log_dir, subset='train')

                    self.eval_one_dataset(sess, self.dataset.test, self.log_dir, subset='test')
                else:
                    print("Input a valid model path.")
//  [python]
def _distribute_eval_model_fn(self, features, labels, run_config):
    start = time.time()
    model = self._model_cls(
        self.model_config,
        self.feature_configs,
        features,
        labels,
        is_training=False)
    predict_dict = model.build_predict_graph()
    loss_dict = model.build_loss_graph()
    loss = tf.add_n(list(loss_dict.values()))
    loss_dict['total_loss'] = loss
    metric_dict = model.build_metric_graph(self.eval_config)
    for loss_key in loss_dict.keys():
      loss_tensor = loss_dict[loss_key]
      # add key-prefix to make loss metric key in the same family of train loss
      metric_dict['loss/loss/' + loss_key] = tf.metrics.mean(loss_tensor)
    tf.logging.info('metric_dict keys: %s' % metric_dict.keys())

    end = time.time()
    tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start))
    metric_name_list = []
    for metric_i in self.eval_config.metrics_set:
      metric_name_list.append(metric_i.WhichOneof('metric'))
    all_var_list = []
    metric_var_list = []
    for var in variables._all_saveable_objects():
      var_name = var.name
      flag = True
      for metric_i in metric_name_list:
        if metric_i in var_name:
          flag = False
          break
      if flag:
        all_var_list.append(var)
      else:
        metric_var_list.append(var)
    global_variables = tf.global_variables()
    metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES)
    model_ready_for_local_init_op = tf.variables_initializer(metric_variables)
    remain_variables = list(
        set(global_variables).difference(set(metric_variables)))
    cur_saver = tf.train.Saver(var_list=remain_variables, sharded=True)
    scaffold = tf.train.Scaffold(
        saver=cur_saver, ready_for_local_init_op=model_ready_for_local_init_op)
    return tf.estimator.EstimatorSpec(
        mode=tf.estimator.ModeKeys.EVAL,
        loss=loss,
        predictions=predict_dict,
        eval_metric_ops=metric_dict,
        scaffold=scaffold)
//  [python]
def test_shared_variables(self):
        vocab, data, options = self._get_vocab_data_options(True, True)
        options['n_epochs'] = 1
        train(options, data, 2, self.tmp_dir, self.tmp_dir)
        self.assertEqual(len(tf.global_variables()), 64)
//  [python]
def __init__(self, vot=False, dis=False):
        self.vot = vot
        self.dis = dis


        config = tf.ConfigProto(allow_soft_placement=True)
        # config.gpu_options.per_process_gpu_memory_fraction = 0.5
        config.gpu_options.allow_growth = True
        self.sess = tf.Session(config=config)

        self.V_image_op = tf.placeholder(tf.float32, (None, 128, 128, 3), name='V_input_image')
        self.V_feat_op = _image_to_feat(self.V_image_op, is_training=False, reuse=False)

        variables = tf.global_variables()  # list
        # Initialize all variables first
        self.sess.run(tf.variables_initializer(variables, name='init'))

        restorer = tf.train.Saver(variables)
        restorer.restore(self.sess, pretrained_model)


        config_file = PROJECT_PATH + 'RPN/ssd_mobilenet_tracking.config'
        checkpoint_dir = PROJECT_PATH + 'RPN/dump'

        model_config, train_config, input_config, eval_config \
            = get_configs_from_pipeline_file(config_file)

        model_scope = 'model'
        model = build_man_model(model_config=model_config, is_training=False)

        self.initConstantOp = tf.placeholder(tf.float32, [1, 1, 1, 512])

        self.initFeatOp, self.initInputOp \
            = build_init_graph(model, model_scope, reuse=None)

        self.pre_box_tensor, self.scores_tensor, self.input_cur_image \
            = build_box_predictor(model, model_scope, self.initConstantOp, reuse=None)


        # self.sess.run(tf.global_variables_initializer())
        variables_to_restore = tf.global_variables()
        restore_model(self.sess, model_scope, checkpoint_dir, variables_to_restore, V_NET)


        with tf.keras.utils.CustomObjectScope({
            'relu6': tf.keras.layers.LeakyReLU(0),
            'DepthwiseConv2D': tf.keras.layers.DepthwiseConv2D}):
            self.branch_z = tf.keras.models.load_model(
                PROJECT_PATH + 'Skim/branch_z_3.h5',
                custom_objects={"tf": tf})
            self.branch_search = tf.keras.models.load_model(
                PROJECT_PATH + 'Skim/branch_search_3.h5',
                custom_objects={"tf": tf})
//  [python]
def restore_model(sess, ckpt_dir):
    """Restore variables from checkpoint dir."""
    checkpoint = tf.train.latest_checkpoint(ckpt_dir)
    ema = tf.train.ExponentialMovingAverage(decay=0.9999)
    ema_vars = tf.trainable_variables() + tf.get_collection('moving_vars')
    for v in tf.all_variables():
        if 'moving_mean' in v.name or 'moving_variance' in v.name:
            ema_vars.append(v)
    ema_vars = list(set(ema_vars))
    var_dict = ema.variables_to_restore(ema_vars)
    saver = tf.train.Saver(max_to_keep=1)
    saver.restore(sess, checkpoint)
//  [python]
def restore_model(sess, ckpt_dir):
    """Restore variables from checkpoint dir."""
    checkpoint = tf.train.latest_checkpoint(ckpt_dir)
    ema = tf.train.ExponentialMovingAverage(decay=0.9999)
    ema_vars = tf.trainable_variables() + tf.get_collection('moving_vars')
    for v in tf.global_variables():
        if 'moving_mean' in v.name or 'moving_variance' in v.name:
            ema_vars.append(v)
    ema_vars = list(set(ema_vars))
    var_dict = ema.variables_to_restore(ema_vars)
    saver = tf.train.Saver(max_to_keep=1)
    saver.restore(sess, checkpoint)
//  [python]
def build(self):
        self.get_data()
        self.inference()
        self.create_model()
        self.eval()
        self.summary()
        self.saver = tf.train.Saver(tf.global_variables())
//  [python]
def testNoBatchNormScaleByDefault(self):
    height, width = 224, 224
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with slim.arg_scope(inception.inception_v1_arg_scope()):
      inception.inception_v1(inputs, num_classes, is_training=False)

    self.assertEqual(tf.global_variables('.*/BatchNorm/gamma:0$'), [])
//  [python]
def evaluate():
    with tf.Graph().as_default():
        
#        log_dir = 'C://Users//kevin//Documents//tensorflow//VGG//logsvgg//train//'
        log_dir = 'C:/Users/kevin/Documents/tensorflow/VGG/logs/train/'
        test_dir = './/data//cifar-10-batches-bin//'
        n_test = 10000
                
        images, labels = input_data.read_cifar10(data_dir=test_dir,
                                                    is_train=False,
                                                    batch_size= BATCH_SIZE,
                                                    shuffle=False)

        logits = VGG.VGG16N(images, N_CLASSES, IS_PRETRAIN)
        correct = tools.num_correct_prediction(logits, labels)
        saver = tf.train.Saver(tf.all_variables())
        
        with tf.Session() as sess:
            
            print("Reading checkpoints...")
            ckpt = tf.train.get_checkpoint_state(log_dir)
            if ckpt and ckpt.model_checkpoint_path:
                global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
                saver.restore(sess, ckpt.model_checkpoint_path)
                print('Loading success, global_step is %s' % global_step)
            else:
                print('No checkpoint file found')
                return
        
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess = sess, coord = coord)
            
            try:
                print('\nEvaluating......')
                num_step = int(math.floor(n_test / BATCH_SIZE))
                num_sample = num_step*BATCH_SIZE
                step = 0
                total_correct = 0
                while step < num_step and not coord.should_stop():
                    batch_correct = sess.run(correct)
                    total_correct += np.sum(batch_correct)
                    step += 1
                print('Total testing samples: %d' %num_sample)
                print('Total correct predictions: %d' %total_correct)
                print('Average accuracy: %.2f%%' %(100*total_correct/num_sample))
            except Exception as e:
                coord.request_stop(e)
            finally:
                coord.request_stop()
                coord.join(threads)
//  [python]
def evaluate():
    with tf.Graph().as_default():
        
#        log_dir = 'C://Users//kevin//Documents//tensorflow//VGG//logsvgg//train//'
        log_dir = 'C:/Users/kevin/Documents/tensorflow/VGG/logs/train/'
        test_dir = './/data//cifar-10-batches-bin//'
        n_test = 10000
                
        images, labels = input_data.read_cifar10(data_dir=test_dir,
                                                    is_train=False,
                                                    batch_size= BATCH_SIZE,
                                                    shuffle=False)

        logits = VGG.VGG16N(images, N_CLASSES, IS_PRETRAIN)
        correct = tools.num_correct_prediction(logits, labels)
        saver = tf.train.Saver(tf.global_variables())
        
        with tf.Session() as sess:
            
            print("Reading checkpoints...")
            ckpt = tf.train.get_checkpoint_state(log_dir)
            if ckpt and ckpt.model_checkpoint_path:
                global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
                saver.restore(sess, ckpt.model_checkpoint_path)
                print('Loading success, global_step is %s' % global_step)
            else:
                print('No checkpoint file found')
                return
        
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess = sess, coord = coord)
            
            try:
                print('\nEvaluating......')
                num_step = int(math.floor(n_test / BATCH_SIZE))
                num_sample = num_step*BATCH_SIZE
                step = 0
                total_correct = 0
                while step < num_step and not coord.should_stop():
                    batch_correct = sess.run(correct)
                    total_correct += np.sum(batch_correct)
                    step += 1
                print('Total testing samples: %d' %num_sample)
                print('Total correct predictions: %d' %total_correct)
                print('Average accuracy: %.2f%%' %(100*total_correct/num_sample))
            except Exception as e:
                coord.request_stop(e)
            finally:
                coord.request_stop()
                coord.join(threads)
//  [python]
def testBatchNormScale(self):
    height, width = 224, 224
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with slim.arg_scope(
        inception.inception_v1_arg_scope(batch_norm_scale=True)):
      inception.inception_v1(inputs, num_classes, is_training=False)

    gamma_names = set(
        v.op.name for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'))
    self.assertGreater(len(gamma_names), 0)
    for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'):
      self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names)
//  [python]
def testBatchNormScale(self):
    height, width = 224, 224
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with slim.arg_scope(
        inception.inception_v1_arg_scope(batch_norm_scale=True)):
      inception.inception_v1(inputs, num_classes, is_training=False)

    gamma_names = set(
        v.op.name for v in tf.global_variables('.*/BatchNorm/gamma:0$'))
    self.assertGreater(len(gamma_names), 0)
    for v in tf.global_variables('.*/BatchNorm/moving_mean:0$'):
      self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names)
//  [python]
def load_checkpoint(sess, checkpoint_path, ras_only=False, perceptual_only=False, gen_model_pretrain=False,
                    train_entire=False):
    if ras_only:
        load_var = {var.op.name: var for var in tf.global_variables() if 'raster_unit' in var.op.name}
    elif perceptual_only:
        load_var = {var.op.name: var for var in tf.global_variables() if 'VGG16' in var.op.name}
    elif train_entire:
        load_var = {var.op.name: var for var in tf.global_variables()
                    if 'discriminator' not in var.op.name
                    and 'raster_unit' not in var.op.name
                    and 'VGG16' not in var.op.name
                    and 'beta1' not in var.op.name
                    and 'beta2' not in var.op.name
                    and 'global_step' not in var.op.name
                    and 'Entire' not in var.op.name
                    }
    else:
        if gen_model_pretrain:
            load_var = {var.op.name: var for var in tf.global_variables()
                        if 'discriminator' not in var.op.name
                        and 'raster_unit' not in var.op.name
                        and 'VGG16' not in var.op.name
                        and 'beta1' not in var.op.name
                        and 'beta2' not in var.op.name
                        # and 'global_step' not in var.op.name
                        }
        else:
            load_var = tf.global_variables()

    restorer = tf.train.Saver(load_var)
    if not ras_only:
        ckpt = tf.train.get_checkpoint_state(checkpoint_path)
        model_checkpoint_path = ckpt.model_checkpoint_path
    else:
        model_checkpoint_path = checkpoint_path
    print('Loading model %s' % model_checkpoint_path)
    restorer.restore(sess, model_checkpoint_path)

    snapshot_step = model_checkpoint_path[model_checkpoint_path.rfind('-') + 1:]
    return snapshot_step
//  [python]
def __init__(self, incoming, num_units, hidden_nonlinearity, horizon=None, hidden_init_trainable=False,
                 **kwargs):
        assert len(incoming.output_shape) == 3
        input_dim = incoming.shape[2]
        gru = tf.nn.rnn_cell.GRUCell(num_units=num_units, activation=hidden_nonlinearity)
        self.num_units = num_units
        self.horizon = horizon
        self.gru = gru
        self.hidden_nonlinearity = hidden_nonlinearity
        Layer.__init__(self, incoming=incoming, **kwargs)
        # dummy input variable
        input_dummy = tf.placeholder(tf.float32, (None, input_dim), "input_dummy")
        hidden_dummy = tf.placeholder(tf.float32, (None, num_units), "hidden_dummy")

        with tf.variable_scope(self.name) as vs:
            gru(input_dummy, hidden_dummy, scope=vs)
            vs.reuse_variables()
            self.scope = vs
            all_vars = [v for v in tf.all_variables() if v.name.startswith(vs.name)]
            trainable_vars = [v for v in tf.trainable_variables() if v.name.startswith(vs.name)]

        for var in trainable_vars:
            self.add_param(spec=var, shape=None, name=None, trainable=True)
        for var in set(all_vars) - set(trainable_vars):
            self.add_param(spec=var, shape=None, name=None, trainable=False)
        self.h0 = self.add_param(tf.zeros_initializer(), (num_units,), name="h0", trainable=hidden_init_trainable,
                                 regularizable=False)
//  [python]
def __init__(self, incoming, num_units, hidden_nonlinearity, horizon=None, hidden_init_trainable=False,
                 **kwargs):
        assert len(incoming.output_shape) == 3
        input_dim = incoming.shape[2]
        gru = tf.nn.rnn_cell.GRUCell(num_units=num_units, activation=hidden_nonlinearity)
        self.num_units = num_units
        self.horizon = horizon
        self.gru = gru
        self.hidden_nonlinearity = hidden_nonlinearity
        Layer.__init__(self, incoming=incoming, **kwargs)
        # dummy input variable
        input_dummy = tf.placeholder(tf.float32, (None, input_dim), "input_dummy")
        hidden_dummy = tf.placeholder(tf.float32, (None, num_units), "hidden_dummy")

        with tf.variable_scope(self.name) as vs:
            gru(input_dummy, hidden_dummy, scope=vs)
            vs.reuse_variables()
            self.scope = vs
            all_vars = [v for v in tf.global_variables() if v.name.startswith(vs.name)]
            trainable_vars = [v for v in tf.trainable_variables() if v.name.startswith(vs.name)]

        for var in trainable_vars:
            self.add_param(spec=var, shape=None, name=None, trainable=True)
        for var in set(all_vars) - set(trainable_vars):
            self.add_param(spec=var, shape=None, name=None, trainable=False)
        self.h0 = self.add_param(tf.zeros_initializer(), (num_units,), name="h0", trainable=hidden_init_trainable,
                                 regularizable=False)
//  [python]
def train(args):
    data_loader = DataLoader(args.batch_size, args.seq_length, args.data_scale)

    if args.model_dir != '' and not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)

    with open(os.path.join(args.model_dir, 'config.pkl'), 'wb') as f:
        pickle.dump(args, f)

    model = Model(args)

    with tf.Session() as sess:
        summary_writer = tf.summary.FileWriter(
            os.path.join(args.model_dir, 'log'), sess.graph)

        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.all_variables())
        for e in range(args.num_epochs):
            sess.run(tf.assign(model.lr,
                               args.learning_rate * (args.decay_rate ** e)))
            data_loader.reset_batch_pointer()
            v_x, v_y = data_loader.validation_data()
            valid_feed = {
                model.input_data: v_x,
                model.target_data: v_y,
                model.state_in: model.state_in.eval()}
            state = model.state_in.eval()
            for b in range(data_loader.num_batches):
                ith_train_step = e * data_loader.num_batches + b
                start = time.time()
                x, y = data_loader.next_batch()
                feed = {
                    model.input_data: x,
                    model.target_data: y,
                    model.state_in: state}
                train_loss_summary, train_loss, state, _ = sess.run(
                    [model.train_loss_summary, model.cost, model.state_out, model.train_op], feed)
                summary_writer.add_summary(train_loss_summary, ith_train_step)

                valid_loss_summary, valid_loss, = sess.run(
                    [model.valid_loss_summary, model.cost], valid_feed)
                summary_writer.add_summary(valid_loss_summary, ith_train_step)

                end = time.time()
                print(
                    "{}/{} (epoch {}), train_loss = {:.3f}, valid_loss = {:.3f}, time/batch = {:.3f}"
                    .format(
                        ith_train_step,
                        args.num_epochs * data_loader.num_batches,
                        e,
                        train_loss,
                        valid_loss,
                        end - start))
                if (ith_train_step %
                        args.save_every == 0) and (ith_train_step > 0):
                    checkpoint_path = os.path.join(
                        args.model_dir, 'model.ckpt')
                    saver.save(
                        sess,
                        checkpoint_path,
                        global_step=ith_train_step)
                    print("model saved to {}".format(checkpoint_path))
//  [python]
def train(args):
    data_loader = DataLoader(args.batch_size, args.seq_length, args.data_scale)

    if args.model_dir != '' and not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)

    with open(os.path.join(args.model_dir, 'config.pkl'), 'wb') as f:
        pickle.dump(args, f)

    model = Model(args)

    with tf.Session() as sess:
        summary_writer = tf.summary.FileWriter(
            os.path.join(args.model_dir, 'log'), sess.graph)

        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.global_variables())
        for e in range(args.num_epochs):
            sess.run(tf.assign(model.lr,
                               args.learning_rate * (args.decay_rate ** e)))
            data_loader.reset_batch_pointer()
            v_x, v_y = data_loader.validation_data()
            valid_feed = {
                model.input_data: v_x,
                model.target_data: v_y,
                model.state_in: model.state_in.eval()}
            state = model.state_in.eval()
            for b in range(data_loader.num_batches):
                ith_train_step = e * data_loader.num_batches + b
                start = time.time()
                x, y = data_loader.next_batch()
                feed = {
                    model.input_data: x,
                    model.target_data: y,
                    model.state_in: state}
                train_loss_summary, train_loss, state, _ = sess.run(
                    [model.train_loss_summary, model.cost, model.state_out, model.train_op], feed)
                summary_writer.add_summary(train_loss_summary, ith_train_step)

                valid_loss_summary, valid_loss, = sess.run(
                    [model.valid_loss_summary, model.cost], valid_feed)
                summary_writer.add_summary(valid_loss_summary, ith_train_step)

                end = time.time()
                print(
                    "{}/{} (epoch {}), train_loss = {:.3f}, valid_loss = {:.3f}, time/batch = {:.3f}"
                    .format(
                        ith_train_step,
                        args.num_epochs * data_loader.num_batches,
                        e,
                        train_loss,
                        valid_loss,
                        end - start))
                if (ith_train_step %
                        args.save_every == 0) and (ith_train_step > 0):
                    checkpoint_path = os.path.join(
                        args.model_dir, 'model.ckpt')
                    saver.save(
                        sess,
                        checkpoint_path,
                        global_step=ith_train_step)
                    print("model saved to {}".format(checkpoint_path))
//  [python]
def __init__(self, incoming, num_units, hidden_nonlinearity, horizon=None, hidden_init_trainable=False,
                 forget_bias=1.0, use_peepholes=False, **kwargs):
        assert not use_peepholes, "Basic LSTM does not support peepholes!"
        assert len(incoming.output_shape) == 3
        input_dim = incoming.shape[2]
        lstm = tf.contrib.rnn.BasicLSTMCell(
            num_units=num_units,
            activation=hidden_nonlinearity,
            state_is_tuple=True,
            forget_bias=forget_bias
        )
        self.num_units = num_units
        self.horizon = horizon
        self.lstm = lstm
        self.hidden_nonlinearity = hidden_nonlinearity
        Layer.__init__(self, incoming=incoming, **kwargs)
        # dummy input variable
        input_dummy = tf.placeholder(tf.float32, (None, input_dim), "input_dummy")
        hidden_dummy = tf.placeholder(tf.float32, (None, num_units), "hidden_dummy")
        cell_dummy = tf.placeholder(tf.float32, (None, num_units), "cell_dummy")

        with tf.variable_scope(self.name) as vs:
            lstm(input_dummy, (cell_dummy, hidden_dummy), scope=vs)
            vs.reuse_variables()
            self.scope = vs
            all_vars = [v for v in tf.all_variables() if v.name.startswith(vs.name)]
            trainable_vars = [v for v in tf.trainable_variables() if v.name.startswith(vs.name)]

        for var in trainable_vars:
            self.add_param(spec=var, shape=None, name=None, trainable=True)
        for var in set(all_vars) - set(trainable_vars):
            self.add_param(spec=var, shape=None, name=None, trainable=False)

        self.h0 = self.add_param(tf.zeros_initializer(), (num_units,), name="h0", trainable=hidden_init_trainable,
                                 regularizable=False)
        self.c0 = self.add_param(tf.zeros_initializer(), (num_units,), name="c0", trainable=hidden_init_trainable,
                                 regularizable=False)
//  [python]
def __init__(self, incoming, num_units, hidden_nonlinearity, horizon=None, hidden_init_trainable=False,
                 forget_bias=1.0, use_peepholes=False, **kwargs):
        assert not use_peepholes, "Basic LSTM does not support peepholes!"
        assert len(incoming.output_shape) == 3
        input_dim = incoming.shape[2]
        lstm = tf.contrib.rnn.BasicLSTMCell(
            num_units=num_units,
            activation=hidden_nonlinearity,
            state_is_tuple=True,
            forget_bias=forget_bias
        )
        self.num_units = num_units
        self.horizon = horizon
        self.lstm = lstm
        self.hidden_nonlinearity = hidden_nonlinearity
        Layer.__init__(self, incoming=incoming, **kwargs)
        # dummy input variable
        input_dummy = tf.placeholder(tf.float32, (None, input_dim), "input_dummy")
        hidden_dummy = tf.placeholder(tf.float32, (None, num_units), "hidden_dummy")
        cell_dummy = tf.placeholder(tf.float32, (None, num_units), "cell_dummy")

        with tf.variable_scope(self.name) as vs:
            lstm(input_dummy, (cell_dummy, hidden_dummy), scope=vs)
            vs.reuse_variables()
            self.scope = vs
            all_vars = [v for v in tf.global_variables() if v.name.startswith(vs.name)]
            trainable_vars = [v for v in tf.trainable_variables() if v.name.startswith(vs.name)]

        for var in trainable_vars:
            self.add_param(spec=var, shape=None, name=None, trainable=True)
        for var in set(all_vars) - set(trainable_vars):
            self.add_param(spec=var, shape=None, name=None, trainable=False)

        self.h0 = self.add_param(tf.zeros_initializer(), (num_units,), name="h0", trainable=hidden_init_trainable,
                                 regularizable=False)
        self.c0 = self.add_param(tf.zeros_initializer(), (num_units,), name="c0", trainable=hidden_init_trainable,
                                 regularizable=False)
//  [python]
def _global_variables():
    return tf.compat.v1.global_variables() if not _IS_TF2 else tf.compat.v1.global_variables()
//  [python]
def _global_variables():
    return tf.global_variables() if not _IS_TF2 else tf.compat.v1.global_variables()
//  [python]
def evaluate(self):
        config = tf.ConfigProto(allow_soft_placement=True)
        ###############
        custom_op = config.graph_options.rewrite_options.custom_optimizers.add()
        custom_op.name = "NpuOptimizer"
        config.graph_options.rewrite_options.remapping = RewriterConfig.OFF
        
        config.gpu_options.per_process_gpu_memory_fraction = 0.7
        with tf.Session(config=config) as sess:
            with tf.device("/gpu:%d" % cfg.GPU_ID):
            # with 1:
                if self.model_path.find('.ckpt') != -1:
                    self.init_opt()
                    print("Reading model parameters from %s" % self.model_path)
                    saver = tf.train.Saver(tf.all_variables())
                    print(tf.all_variables())
                    saver.restore(sess, self.model_path)

                    # self.eval_one_dataset(sess, self.dataset.train, self.log_dir, subset='train')

                    self.eval_one_dataset(sess, self.dataset.test, self.log_dir, subset='test')
                else:
                    print("Input a valid model path.")
//  [python]
def evaluate(self):
        config = tf.ConfigProto(allow_soft_placement=True)
        ###############
        custom_op = config.graph_options.rewrite_options.custom_optimizers.add()
        custom_op.name = "NpuOptimizer"
        config.graph_options.rewrite_options.remapping = RewriterConfig.OFF
        
        config.gpu_options.per_process_gpu_memory_fraction = 0.7
        with tf.Session(config=config) as sess:
            with tf.device("/gpu:%d" % cfg.GPU_ID):
            # with 1:
                if self.model_path.find('.ckpt') != -1:
                    self.init_opt()
                    print("Reading model parameters from %s" % self.model_path)
                    saver = tf.train.Saver(tf.global_variables())
                    print(tf.global_variables())
                    saver.restore(sess, self.model_path)

                    # self.eval_one_dataset(sess, self.dataset.train, self.log_dir, subset='train')

                    self.eval_one_dataset(sess, self.dataset.test, self.log_dir, subset='test')
                else:
                    print("Input a valid model path.")
//  [python]
def load_checkpoints(sess, var_scopes = ('encoder', 'decoder', 'dense')):

  checkpoint_path =  config.lip_model_path
  if checkpoint_path:
    if os.path.isdir(checkpoint_path):
      checkpoint = tf.train.latest_checkpoint(checkpoint_path)
    else:
      checkpoint = checkpoint_path

  if config.featurizer:

    if checkpoint_path:
      from tensorflow.contrib.framework.python.framework import checkpoint_utils
      var_list = checkpoint_utils.list_variables(checkpoint)
      for var in var_list:
        if 'visual_frontend' in var[0]:
          var_scopes = var_scopes + ('visual_frontend',)
          break

    if not 'visual_frontend' in var_scopes:
      featurizer_vars = tf.all_variables(scope='visual_frontend')
      featurizer_ckpt = tf.train.get_checkpoint_state(config.featurizer_model_path)
      featurizer_vars = [var for var in featurizer_vars if not 'Adam' in var.name]
      tf.train.Saver(featurizer_vars).restore(sess, featurizer_ckpt.model_checkpoint_path)

  all_variables = []
  for scope in var_scopes:
    all_variables += [var for var in tf.all_variables(scope=scope)
                      if not 'Adam' in var.name ]
  if checkpoint_path:
    tf.train.Saver(all_variables).restore(sess, checkpoint)

    print("Restored saved model {}!".format(checkpoint))
//  [python]
def load_checkpoints(sess, var_scopes = ('encoder', 'decoder', 'dense')):

  checkpoint_path =  config.lip_model_path
  if checkpoint_path:
    if os.path.isdir(checkpoint_path):
      checkpoint = tf.train.latest_checkpoint(checkpoint_path)
    else:
      checkpoint = checkpoint_path

  if config.featurizer:

    if checkpoint_path:
      from tensorflow.contrib.framework.python.framework import checkpoint_utils
      var_list = checkpoint_utils.list_variables(checkpoint)
      for var in var_list:
        if 'visual_frontend' in var[0]:
          var_scopes = var_scopes + ('visual_frontend',)
          break

    if not 'visual_frontend' in var_scopes:
      featurizer_vars = tf.global_variables(scope='visual_frontend')
      featurizer_ckpt = tf.train.get_checkpoint_state(config.featurizer_model_path)
      featurizer_vars = [var for var in featurizer_vars if not 'Adam' in var.name]
      tf.train.Saver(featurizer_vars).restore(sess, featurizer_ckpt.model_checkpoint_path)

  all_variables = []
  for scope in var_scopes:
    all_variables += [var for var in tf.global_variables(scope=scope)
                      if not 'Adam' in var.name ]
  if checkpoint_path:
    tf.train.Saver(all_variables).restore(sess, checkpoint)

    print("Restored saved model {}!".format(checkpoint))
//  [python]
def train(fps, args):
  with tf.name_scope('loader'):
    dataset = tf.data.TFRecordDataset(fps)
    dataset = dataset.map(lambda x: record_to_xy(x, args.data_labels))
    dataset = dataset.shuffle(buffer_size=8192)
    dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(args.train_batch_size))
    dataset = dataset.repeat()
    iterator = dataset.make_one_shot_iterator()

    x, y = iterator.get_next()

  with tf.variable_scope('classifier'):
    logits = model(x, len(args.data_labels), train=True)
  for v in tf.all_variables():
    print(v.get_shape(), v.name)

  xent = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)
  xent_mean = tf.reduce_mean(xent)

  opt = tf.train.GradientDescentOptimizer(1e-4)
  train_op = opt.minimize(xent_mean, global_step=tf.train.get_or_create_global_step())

  preds = tf.argmax(logits, axis=1, output_type=tf.int32)
  acc = tf.reduce_mean(tf.cast(tf.equal(preds, y), tf.float32))

  tf.summary.audio('x', tf.expand_dims(x, axis=2), 16000)
  tf.summary.scalar('xent', xent_mean)
  tf.summary.scalar('acc', acc)
  tf.summary.histogram('xent', xent_mean)

  with tf.train.MonitoredTrainingSession(
      checkpoint_dir=args.train_dir,
      save_checkpoint_secs=args.train_save_secs,
      save_summaries_secs=args.train_summary_secs) as sess:
    while True:
      _, _acc = sess.run([train_op, acc])
//  [python]
def train(fps, args):
  with tf.name_scope('loader'):
    dataset = tf.data.TFRecordDataset(fps)
    dataset = dataset.map(lambda x: record_to_xy(x, args.data_labels))
    dataset = dataset.shuffle(buffer_size=8192)
    dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(args.train_batch_size))
    dataset = dataset.repeat()
    iterator = dataset.make_one_shot_iterator()

    x, y = iterator.get_next()

  with tf.variable_scope('classifier'):
    logits = model(x, len(args.data_labels), train=True)
  for v in tf.global_variables():
    print(v.get_shape(), v.name)

  xent = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)
  xent_mean = tf.reduce_mean(xent)

  opt = tf.train.GradientDescentOptimizer(1e-4)
  train_op = opt.minimize(xent_mean, global_step=tf.train.get_or_create_global_step())

  preds = tf.argmax(logits, axis=1, output_type=tf.int32)
  acc = tf.reduce_mean(tf.cast(tf.equal(preds, y), tf.float32))

  tf.summary.audio('x', tf.expand_dims(x, axis=2), 16000)
  tf.summary.scalar('xent', xent_mean)
  tf.summary.scalar('acc', acc)
  tf.summary.histogram('xent', xent_mean)

  with tf.train.MonitoredTrainingSession(
      checkpoint_dir=args.train_dir,
      save_checkpoint_secs=args.train_save_secs,
      save_summaries_secs=args.train_summary_secs) as sess:
    while True:
      _, _acc = sess.run([train_op, acc])
//  [python]
def restore_variables(sess):

    # Fresh train directly from ImageNet weights
    print('Loading initial model weights from {:s}'.format(pretrained_model))

    variables = tf.all_variables() # list
    # Initialize all variables first
    sess.run(tf.variables_initializer(variables, name='init'))
    var_keep_dic = get_variables_in_checkpoint_file(pretrained_model)
    # Get the variables to restore, ignoring the variables to fix
    variables_to_restore = get_variables_to_restore(variables, var_keep_dic)

    restorer = tf.train.Saver(variables_to_restore)
    restorer.restore(sess, pretrained_model)
    print('Loaded.')
    # Need to fix the variables before loading, so that the RGB weights are changed to BGR
    # For VGG16 it also changes the convolutional weights fc6 and fc7 to
    # fully connected weights
    fix_variables(sess, pretrained_model)
    print('Fixed.')
//  [python]
def restore_variables(sess):

    # Fresh train directly from ImageNet weights
    print('Loading initial model weights from {:s}'.format(pretrained_model))

    variables = tf.global_variables() # list
    # Initialize all variables first
    sess.run(tf.variables_initializer(variables, name='init'))
    var_keep_dic = get_variables_in_checkpoint_file(pretrained_model)
    # Get the variables to restore, ignoring the variables to fix
    variables_to_restore = get_variables_to_restore(variables, var_keep_dic)

    restorer = tf.train.Saver(variables_to_restore)
    restorer.restore(sess, pretrained_model)
    print('Loaded.')
    # Need to fix the variables before loading, so that the RGB weights are changed to BGR
    # For VGG16 it also changes the convolutional weights fc6 and fc7 to
    # fully connected weights
    fix_variables(sess, pretrained_model)
    print('Fixed.')
//  [python]
def initialize_uninitialized(self, ):
        """Only initializes the variables of a TensorFlow session that were not
        already initialized.
        """
        # List all global variables.
        sess = self.sess
        global_vars = tf.global_variables()

        # Find initialized status for all variables.
        is_var_init = [tf.is_variable_initialized(var) for var in global_vars]
        is_initialized = sess.run(is_var_init)

        # List all variables that were not previously initialized.
        not_initialized_vars = [var for (var, init) in
                                zip(global_vars, is_initialized) if not init]
        for v in not_initialized_vars:
            print('[!] not init: {}'.format(v.name))
        # Initialize all uninitialized variables found, if any.
        if len(not_initialized_vars):
            sess.run(tf.variables_initializer(not_initialized_vars))