jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_quantize_lstm_deepspeech_time_major_false_quantsim_and_forward_pass(self):
        """ Test connected graph construction on a model with lstm op """
        tf.reset_default_graph()

        sess = tf.Session()
        np.random.seed(0)
        tf.set_random_seed(0)

        with sess.graph.as_default():
            inputs = tf.keras.Input(shape=(3, 100))

            # Add a LSTM layer with 12 internal units.
            x, state_h, state_c = tf.keras.layers.LSTM(12, return_state=True,
                                                       return_sequences=True,
                                                       name='lstm_stacked')(inputs)
            x2 = tf.keras.layers.LSTM(12, name='last_lstm')(x)
            _ = tf.keras.layers.Dense(12, activation=tf.nn.softmax,
                                      name="lstm_model")(x2)

        init = tf.global_variables_initializer()
        sess.run(init)

        sim = QuantizationSimModel(sess, ['input_1'], ['lstm_model/Softmax'],
                                   use_cuda=False)

        # validate quantsim
        # get ops and make sure we have a quantized op added to the conditional block
        quantized_graph_op_names = self._get_quant_ops_from_tf_graph(sim.session.graph)

        # _ = tf.summary.FileWriter('./lstm_tm', sess.graph)
        self.validate_internal_lstm_quantisim_nodes(quantized_graph_op_names, 'lstm_stacked', True, False)
        self.validate_internal_lstm_quantisim_nodes(quantized_graph_op_names, 'last_lstm')

        # validate forward pass
        self.validate_general_lstm_forward_pass_and_encoding(sess, sim, 9, 14)
        self.validate_general_lstm_forward_pass_and_encoding(sess, sim, 9, 14)

        # close tf sessions
        sess.close()
        sim.session.close()
        del sim
//  [python]
def initialize_session(sess):

    saver = tf.train.Saver(max_to_keep=FLAGS.max_to_keep)

    init_op = tf.group(tf.initialize_all_variables(),
                       tf.local_variables_initializer())
    sess.run(init_op)

    lu.print_initialize()

    return saver
//  [python]
def initialize_session(sess):

    saver = tf.train.Saver(max_to_keep=FLAGS.max_to_keep)

    init_op = tf.group(tf.global_variables_initializer(),
                       tf.local_variables_initializer())
    sess.run(init_op)

    lu.print_initialize()

    return saver
//  [python]
def main(_):
    tf.compat.v1.reset_default_graph()
    print('build model')
    '''
    #GPU版本
    gpu_options = tf.compat.v1.GPUOptions(visible_device_list=FLAGS.cuda, allow_growth=True)
    with tf.Graph().as_default():
        set_seed()
        sess = tf.compat.v1.Session(
            config=tf.compat.v1.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True, intra_op_parallelism_threads=1, inter_op_parallelism_threads=1))
        with sess.as_default():
            initializer = tf.contrib.layers.xavier_initializer()
            with tf.variable_scope('', initializer=initializer):
                model = Baseline(FLAGS)
            sess.run(tf.compat.v1.global_variables_initializer())
            saver = tf.compat.v1.train.Saver(max_to_keep=None)
            model.run_model(sess, saver)
    '''
    with tf.Graph().as_default():
         set_seed()
         sess = tf.compat.v1.Session()
         with sess.as_default():
            initializer = tf.contrib.layers.xavier_initializer()
            with tf.compat.v1.variable_scope('', initializer=initializer):
                model = Baseline(FLAGS)
            sess.run(tf.compat.v1.global_variables_initializer())
            saver = tf.compat.v1.train.Saver(max_to_keep=None)
            model.run_model(sess, saver)
//  [python]
def main(_):
    tf.compat.v1.reset_default_graph()
    print('build model')
    '''
    #GPU版本
    gpu_options = tf.compat.v1.GPUOptions(visible_device_list=FLAGS.cuda, allow_growth=True)
    with tf.Graph().as_default():
        set_seed()
        sess = tf.compat.v1.Session(
            config=tf.compat.v1.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True, intra_op_parallelism_threads=1, inter_op_parallelism_threads=1))
        with sess.as_default():
            initializer = tf.contrib.layers.xavier_initializer()
            with tf.variable_scope('', initializer=initializer):
                model = Baseline(FLAGS)
            sess.run(tf.global_variables_initializer())
            saver = tf.compat.v1.train.Saver(max_to_keep=None)
            model.run_model(sess, saver)
    '''
    with tf.Graph().as_default():
         set_seed()
         sess = tf.compat.v1.Session()
         with sess.as_default():
            initializer = tf.contrib.layers.xavier_initializer()
            with tf.compat.v1.variable_scope('', initializer=initializer):
                model = Baseline(FLAGS)
            sess.run(tf.global_variables_initializer())
            saver = tf.compat.v1.train.Saver(max_to_keep=None)
            model.run_model(sess, saver)
//  [python]
def create_vggish_frozen_graph():
  """Create the VGGish frozen graph."""
  os.system('git clone https://github.com/tensorflow/models.git')
  sys.path.append('models/research/audioset/vggish/')

  import vggish_slim
  os.system('curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt')
  ckpt_path = 'vggish_model.ckpt'

  with tf.Graph().as_default(), tf.Session() as sess:
    vggish_slim.define_vggish_slim(training=False)
    vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path)

    saver = tf.train.Saver(tf.all_variables())

    freeze_graph.freeze_graph_with_def_protos(
        sess.graph_def,
        saver.as_saver_def(),
        ckpt_path,
        'vggish/fc2/BiasAdd',
        restore_op_name=None,
        filename_tensor_name=None,
        output_graph='/tmp/mediapipe/vggish_new.pb',
        clear_devices=True,
        initializer_nodes=None)
  os.system('rm -rf models/')
  os.system('rm %s' % ckpt_path)
//  [python]
def create_vggish_frozen_graph():
  """Create the VGGish frozen graph."""
  os.system('git clone https://github.com/tensorflow/models.git')
  sys.path.append('models/research/audioset/vggish/')

  import vggish_slim
  os.system('curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt')
  ckpt_path = 'vggish_model.ckpt'

  with tf.Graph().as_default(), tf.Session() as sess:
    vggish_slim.define_vggish_slim(training=False)
    vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path)

    saver = tf.train.Saver(tf.global_variables())

    freeze_graph.freeze_graph_with_def_protos(
        sess.graph_def,
        saver.as_saver_def(),
        ckpt_path,
        'vggish/fc2/BiasAdd',
        restore_op_name=None,
        filename_tensor_name=None,
        output_graph='/tmp/mediapipe/vggish_new.pb',
        clear_devices=True,
        initializer_nodes=None)
  os.system('rm -rf models/')
  os.system('rm %s' % ckpt_path)
//  [python]
def test_reuse_model(self, avg_pool, is_training):
    batch_size = 2
    num_classes = 10
    images = _construct_images(batch_size)
    config = bagnet_config.get_config()
    config.num_classes = num_classes
    config.avg_pool = avg_pool
    model = bagnet_model.BagNet(config)
    # Build once.
    logits1, _ = model(images, is_training)
    num_params = len(tf.all_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice.
    logits2, _ = model(
        images, is_training)
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.all_variables(), num_params)
    init = tf.global_variables_initializer()
    self.evaluate(init)
    logits1, logits2 = self.evaluate((logits1, logits2))
    l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
    np.testing.assert_almost_equal(logits1, logits2, decimal=9)
    np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
//  [python]
def test_reuse_model(self, avg_pool, is_training):
    batch_size = 2
    num_classes = 10
    images = _construct_images(batch_size)
    config = bagnet_config.get_config()
    config.num_classes = num_classes
    config.avg_pool = avg_pool
    model = bagnet_model.BagNet(config)
    # Build once.
    logits1, _ = model(images, is_training)
    num_params = len(tf.global_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice.
    logits2, _ = model(
        images, is_training)
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.global_variables(), num_params)
    init = tf.global_variables_initializer()
    self.evaluate(init)
    logits1, logits2 = self.evaluate((logits1, logits2))
    l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
    np.testing.assert_almost_equal(logits1, logits2, decimal=9)
    np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
//  [python]
def test_reuse_model(self, policy, is_training, classnet_type):
    config = saccader_classnet_config.get_config()
    num_times = 2
    image_shape = (100, 100, 3)
    num_classes = 10
    config.num_classes = num_classes
    config.num_times = num_times
    config.classnet_type = classnet_type
    batch_size = 3
    images = tf.constant(
        np.random.rand(*((batch_size,) + image_shape)), dtype=tf.float32)
    model = saccader_classnet.SaccaderClassNet(config)
    logits1 = model(images, images, num_times=num_times,
                    is_training_saccader=is_training,
                    is_training_classnet=is_training,
                    policy=policy)[0]
    num_params = len(tf.all_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice with different num_times.
    logits2 = model(images, images, num_times=num_times+1,
                    is_training_saccader=is_training,
                    is_training_classnet=is_training,
                    policy=policy)[0]
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.all_variables(), num_params)
    init = tf.global_variables_initializer()
    self.evaluate(init)
    logits1, logits2 = self.evaluate((logits1, logits2))
    l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
    np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=5)
//  [python]
def test_reuse_model(self, policy, is_training, classnet_type):
    config = saccader_classnet_config.get_config()
    num_times = 2
    image_shape = (100, 100, 3)
    num_classes = 10
    config.num_classes = num_classes
    config.num_times = num_times
    config.classnet_type = classnet_type
    batch_size = 3
    images = tf.constant(
        np.random.rand(*((batch_size,) + image_shape)), dtype=tf.float32)
    model = saccader_classnet.SaccaderClassNet(config)
    logits1 = model(images, images, num_times=num_times,
                    is_training_saccader=is_training,
                    is_training_classnet=is_training,
                    policy=policy)[0]
    num_params = len(tf.global_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice with different num_times.
    logits2 = model(images, images, num_times=num_times+1,
                    is_training_saccader=is_training,
                    is_training_classnet=is_training,
                    policy=policy)[0]
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.global_variables(), num_params)
    init = tf.global_variables_initializer()
    self.evaluate(init)
    logits1, logits2 = self.evaluate((logits1, logits2))
    l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
    np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=5)
//  [python]
def __init__(self, model, steps=0, epochs=0, restore_snap=None):

        self.training_step = steps
        self.training_epoch = epochs
        #current_lr = model.config.learning_rate * model.config.lr_decays[self.training_epoch]
        #print(current_lr)
        # Add training ops
        self.add_train_ops(model)
        # Tensorflow Saver definition
        my_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='KernelPointNetwork')
        self.saver = tf.train.Saver(my_vars, max_to_keep=100)

        """
        print('*************************************')
        sum = 0
        for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='KernelPointNetwork'):
            #print(var.name, var.shape)
            sum += np.prod(var.shape)
        print('total parameters : ', sum)
        print('*************************************')

        print('*************************************')
        sum = 0
        for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='KernelPointNetwork'):
            #print(var.name, var.shape)
            sum += np.prod(var.shape)
        print('total parameters : ', sum)
        print('*************************************')
        """

        # Create a session for running Ops on the Graph.
        on_CPU = False
        if on_CPU:
            cProto = tf.ConfigProto(device_count={'GPU': 0})
        else:
            cProto = tf.ConfigProto()
            cProto.gpu_options.allow_growth = True
        self.sess = tf.Session(config=cProto)

        # Init variables
        self.sess.run(tf.initialize_all_variables())





        # Name of the snapshot to restore to (None if you want to start from beginning)
        # restore_snap = join(self.saving_path, 'snapshots/snap-40000')
        if (restore_snap is not None):
            exclude_vars = ['softmax', 'head_unary_conv', '/fc/']
            restore_vars = my_vars
            for exclude_var in exclude_vars:
                restore_vars = [v for v in restore_vars if exclude_var not in v.name]
            restorer = tf.train.Saver(restore_vars)
            restorer.restore(self.sess, restore_snap)
            print("Model restored.")
//  [python]
def __init__(self, model, steps=0, epochs=0, restore_snap=None):

        self.training_step = steps
        self.training_epoch = epochs
        #current_lr = model.config.learning_rate * model.config.lr_decays[self.training_epoch]
        #print(current_lr)
        # Add training ops
        self.add_train_ops(model)
        # Tensorflow Saver definition
        my_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='KernelPointNetwork')
        self.saver = tf.train.Saver(my_vars, max_to_keep=100)

        """
        print('*************************************')
        sum = 0
        for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='KernelPointNetwork'):
            #print(var.name, var.shape)
            sum += np.prod(var.shape)
        print('total parameters : ', sum)
        print('*************************************')

        print('*************************************')
        sum = 0
        for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='KernelPointNetwork'):
            #print(var.name, var.shape)
            sum += np.prod(var.shape)
        print('total parameters : ', sum)
        print('*************************************')
        """

        # Create a session for running Ops on the Graph.
        on_CPU = False
        if on_CPU:
            cProto = tf.ConfigProto(device_count={'GPU': 0})
        else:
            cProto = tf.ConfigProto()
            cProto.gpu_options.allow_growth = True
        self.sess = tf.Session(config=cProto)

        # Init variables
        self.sess.run(tf.global_variables_initializer())





        # Name of the snapshot to restore to (None if you want to start from beginning)
        # restore_snap = join(self.saving_path, 'snapshots/snap-40000')
        if (restore_snap is not None):
            exclude_vars = ['softmax', 'head_unary_conv', '/fc/']
            restore_vars = my_vars
            for exclude_var in exclude_vars:
                restore_vars = [v for v in restore_vars if exclude_var not in v.name]
            restorer = tf.train.Saver(restore_vars)
            restorer.restore(self.sess, restore_snap)
            print("Model restored.")
//  [python]
def get_load_vars():
  """Returns a list of variables to load from the checkpoint."""
  tf.train.get_or_create_global_step()
  load_vars = []
  for v in tf.all_variables():
    if v.name.startswith('augmentation'):
      continue
    if 'train_images' in v.name or 'train_labels' in v.name:
      continue
    load_vars.append(v)
  return load_vars
//  [python]
def get_load_vars():
  """Returns a list of variables to load from the checkpoint."""
  tf.train.get_or_create_global_step()
  load_vars = []
  for v in tf.global_variables():
    if v.name.startswith('augmentation'):
      continue
    if 'train_images' in v.name or 'train_labels' in v.name:
      continue
    load_vars.append(v)
  return load_vars
//  [python]
def test_saccader_pretrain_loss(self):
    batch_size = 2
    num_classes = 1001
    images = tf.random_uniform(
        shape=(batch_size, 224, 224, 3), minval=-1, maxval=1, dtype=tf.float32)
    config = saccader_config.get_config()
    config.num_classes = num_classes

    model = saccader.Saccader(config)

    model(
        images,
        num_times=6,
        is_training=True,
        policy="learned")

    num_params = len(tf.all_variables())

    pretrain_loss = losses.saccader_pretraining_loss(
        model, images, is_training=True)

    # Test loss does not introduce new variables.
    self.assertLen(tf.all_variables(), num_params)

    # Gradients with respect to location variables should exist.
    for v, g in zip(model.var_list_location, tf.gradients(
        pretrain_loss, model.var_list_location)):
      if v.trainable:
        self.assertIsNotNone(g)

    # Gradients with respect to classification variables should be None.
    for g in tf.gradients(
        pretrain_loss, model.var_list_classification):
      self.assertIsNone(g)

    # Test evaluating the loss
    self.evaluate(tf.global_variables_initializer())
    self.evaluate(pretrain_loss)
//  [python]
def test_saccader_pretrain_loss(self):
    batch_size = 2
    num_classes = 1001
    images = tf.random_uniform(
        shape=(batch_size, 224, 224, 3), minval=-1, maxval=1, dtype=tf.float32)
    config = saccader_config.get_config()
    config.num_classes = num_classes

    model = saccader.Saccader(config)

    model(
        images,
        num_times=6,
        is_training=True,
        policy="learned")

    num_params = len(tf.global_variables())

    pretrain_loss = losses.saccader_pretraining_loss(
        model, images, is_training=True)

    # Test loss does not introduce new variables.
    self.assertLen(tf.global_variables(), num_params)

    # Gradients with respect to location variables should exist.
    for v, g in zip(model.var_list_location, tf.gradients(
        pretrain_loss, model.var_list_location)):
      if v.trainable:
        self.assertIsNotNone(g)

    # Gradients with respect to classification variables should be None.
    for g in tf.gradients(
        pretrain_loss, model.var_list_classification):
      self.assertIsNone(g)

    # Test evaluating the loss
    self.evaluate(tf.global_variables_initializer())
    self.evaluate(pretrain_loss)
//  [python]
def variables_with_scope(scope):
  #scope is a top scope here, otherwise change startswith part
  return [v for v in tf.all_variables() if v.name.startswith(scope)]
//  [python]
def variables_with_scope(scope):
  return [v for v in tf.global_variables() if v.name.startswith(scope)]
//  [python]
def test_reuse_model(self, rank, is_training, special_type):
    batch_size = 5
    num_classes = 10
    num_channels = 3
    images = _construct_images(batch_size)
    config = simple_model_config.get_config()
    config.num_classes = num_classes
    config.num_channels = num_channels
    config.batch_norm = False
    config.kernel_size_list = [3, 3, 3]
    config.num_filters_list = [64, 64, 64]
    config.strides_list = [2, 2, 1]
    config.layer_types = ['conv2d', special_type, 'conv2d']
    config.rank = rank

    model = simple_model.SimpleNetwork(config)

    # Build once.
    logits1, _ = model(images, is_training)
    num_params = len(tf.all_variables())
    # Build twice.
    logits2, _ = model(images, is_training)
    # Ensure variables are reused.
    self.assertLen(tf.all_variables(), num_params)
    init = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init)
      # Ensure operations are the same after reuse.
      err_logits = (np.abs(sess.run(logits1 - logits2))).sum()
      self.assertAlmostEqual(err_logits, 0, 9)
//  [python]
def test_reuse_model(self, rank, is_training, special_type):
    batch_size = 5
    num_classes = 10
    num_channels = 3
    images = _construct_images(batch_size)
    config = simple_model_config.get_config()
    config.num_classes = num_classes
    config.num_channels = num_channels
    config.batch_norm = False
    config.kernel_size_list = [3, 3, 3]
    config.num_filters_list = [64, 64, 64]
    config.strides_list = [2, 2, 1]
    config.layer_types = ['conv2d', special_type, 'conv2d']
    config.rank = rank

    model = simple_model.SimpleNetwork(config)

    # Build once.
    logits1, _ = model(images, is_training)
    num_params = len(tf.global_variables())
    # Build twice.
    logits2, _ = model(images, is_training)
    # Ensure variables are reused.
    self.assertLen(tf.global_variables(), num_params)
    init = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init)
      # Ensure operations are the same after reuse.
      err_logits = (np.abs(sess.run(logits1 - logits2))).sum()
      self.assertAlmostEqual(err_logits, 0, 9)
//  [python]
def testBatchNormIsTraining(self, is_training):
    feature_dims = (128, 128)
    inputs = tf.random.uniform((3, 4), dtype=tf.float64, seed=1)
    projection_head = projection_head_lib.ProjectionHead(
        feature_dims=feature_dims, use_batch_norm=True)
    outputs = projection_head(inputs, training=is_training)
    statistics_vars = [
        var for var in tf.all_variables() if 'moving_' in var.name
    ]
    self.assertLen(statistics_vars, 2)
    grads = tf.gradients(outputs, statistics_vars)
    self.assertLen(grads, 2)
    if is_training:
      self.assertAllEqual([None, None], grads)
      self.assertTrue(tf.get_collection(tf.GraphKeys.UPDATE_OPS))
    else:
      self.assertNotIn(None, grads)
//  [python]
def testBatchNormIsTraining(self, is_training):
    feature_dims = (128, 128)
    inputs = tf.random.uniform((3, 4), dtype=tf.float64, seed=1)
    projection_head = projection_head_lib.ProjectionHead(
        feature_dims=feature_dims, use_batch_norm=True)
    outputs = projection_head(inputs, training=is_training)
    statistics_vars = [
        var for var in tf.global_variables() if 'moving_' in var.name
    ]
    self.assertLen(statistics_vars, 2)
    grads = tf.gradients(outputs, statistics_vars)
    self.assertLen(grads, 2)
    if is_training:
      self.assertAllEqual([None, None], grads)
      self.assertTrue(tf.get_collection(tf.GraphKeys.UPDATE_OPS))
    else:
      self.assertNotIn(None, grads)
//  [python]
def test_reuse_model(self, is_training):
    batch_size = 5
    images = _construct_images(batch_size)
    config = mnist_config.ConfigDict()

    model = mnist_model.MNISTNetwork(config)

    # Build once.
    logits1, _ = model(images, is_training)
    num_params = len(tf.all_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice.
    logits2, _ = model(images, is_training)
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.all_variables(), num_params)
    init = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init)
      # Ensure operations are the same after reuse.
      err_logits = (np.abs(sess.run(logits1 - logits2))).sum()
      self.assertAlmostEqual(err_logits, 0, 9)
      err_losses = (np.abs(sess.run(l2_loss1 - l2_loss2))).sum()
      self.assertAlmostEqual(err_losses, 0, 9)
//  [python]
def test_reuse_model(self, is_training):
    batch_size = 5
    images = _construct_images(batch_size)
    config = mnist_config.ConfigDict()

    model = mnist_model.MNISTNetwork(config)

    # Build once.
    logits1, _ = model(images, is_training)
    num_params = len(tf.global_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice.
    logits2, _ = model(images, is_training)
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.global_variables(), num_params)
    init = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init)
      # Ensure operations are the same after reuse.
      err_logits = (np.abs(sess.run(logits1 - logits2))).sum()
      self.assertAlmostEqual(err_logits, 0, 9)
      err_losses = (np.abs(sess.run(l2_loss1 - l2_loss2))).sum()
      self.assertAlmostEqual(err_losses, 0, 9)
//  [python]
def test_reuse_model(self, policy, is_training):
    config = saccader_config.get_config()
    num_times = 2
    image_shape = (224, 224, 3)
    num_classes = 10
    config.num_classes = num_classes
    config.num_times = num_times
    batch_size = 3
    images1 = tf.constant(
        np.random.rand(*((batch_size,) + image_shape)), dtype=tf.float32)
    model = saccader.Saccader(config)
    logits1 = model(images1, num_times=num_times, is_training=is_training,
                    policy=policy)[0]
    num_params = len(tf.all_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice with different num_times and different batch size.
    images2 = tf.constant(
        np.random.rand(*((batch_size - 1,) + image_shape)), dtype=tf.float32)
    logits2 = model(images2, num_times=num_times+1, is_training=is_training,
                    policy=policy)[0]
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.all_variables(), num_params)
    init = tf.global_variables_initializer()
    self.evaluate(init)
    logits1, logits2 = self.evaluate((logits1, logits2))
    l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
    np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
//  [python]
def test_reuse_model(self, policy, is_training):
    config = saccader_config.get_config()
    num_times = 2
    image_shape = (224, 224, 3)
    num_classes = 10
    config.num_classes = num_classes
    config.num_times = num_times
    batch_size = 3
    images1 = tf.constant(
        np.random.rand(*((batch_size,) + image_shape)), dtype=tf.float32)
    model = saccader.Saccader(config)
    logits1 = model(images1, num_times=num_times, is_training=is_training,
                    policy=policy)[0]
    num_params = len(tf.global_variables())
    l2_loss1 = tf.losses.get_regularization_loss()
    # Build twice with different num_times and different batch size.
    images2 = tf.constant(
        np.random.rand(*((batch_size - 1,) + image_shape)), dtype=tf.float32)
    logits2 = model(images2, num_times=num_times+1, is_training=is_training,
                    policy=policy)[0]
    l2_loss2 = tf.losses.get_regularization_loss()

    # Ensure variables are reused.
    self.assertLen(tf.global_variables(), num_params)
    init = tf.global_variables_initializer()
    self.evaluate(init)
    logits1, logits2 = self.evaluate((logits1, logits2))
    l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
    np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
//  [python]
def clean_ckpt(_):
  input_ckpt = FLAGS.clean_input_ckpt
  output_model_dir = FLAGS.clean_output_model_dir

  tf.reset_default_graph()

  var_list = tf.contrib.framework.list_variables(input_ckpt)
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step") and "adam" not in name.lower():
      var_values[name] = None
      tf.logging.info("Include {}".format(name))
    else:
      tf.logging.info("Exclude {}".format(name))

  tf.logging.info("Loading from {}".format(input_ckpt))
  reader = tf.contrib.framework.load_checkpoint(input_ckpt)
  for name in var_values:
    tensor = reader.get_tensor(name)
    var_dtypes[name] = tensor.dtype
    var_values[name] = tensor

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  if not tf.gfile.Exists(output_model_dir):
    tf.gfile.MakeDirs(output_model_dir)

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})

    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
               global_step=global_step)
//  [python]
def clean_ckpt(_):
  input_ckpt = FLAGS.clean_input_ckpt
  output_model_dir = FLAGS.clean_output_model_dir

  tf.reset_default_graph()

  var_list = tf.contrib.framework.list_variables(input_ckpt)
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step") and "adam" not in name.lower():
      var_values[name] = None
      tf.logging.info("Include {}".format(name))
    else:
      tf.logging.info("Exclude {}".format(name))

  tf.logging.info("Loading from {}".format(input_ckpt))
  reader = tf.contrib.framework.load_checkpoint(input_ckpt)
  for name in var_values:
    tensor = reader.get_tensor(name)
    var_dtypes[name] = tensor.dtype
    var_values[name] = tensor

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.global_variables())

  if not tf.gfile.Exists(output_model_dir):
    tf.gfile.MakeDirs(output_model_dir)

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})

    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
               global_step=global_step)
//  [python]
def total_params(self):
        total_parameters = 0
        for variable in tf.all_variables():
            # shape is an array of tf.Dimension
            shape = variable.get_shape()
            print(shape)
            print(len(shape))
            variable_parameters = 1
            for dim in shape:
                print(dim)
                variable_parameters *= dim.value
            print(variable_parameters)
            total_parameters += variable_parameters
        print('Total Parameters=%s', str(total_parameters))
//  [python]
def total_params(self):
        total_parameters = 0
        for variable in tf.global_variables():
            shape = variable.get_shape()
            variable_parameters = 1
            for dim in shape:
                variable_parameters *= dim.value
            total_parameters += variable_parameters
        print('Total Parameters=%s', str(total_parameters))
//  [python]
def avg_checkpoints(model_dir, output_model_dir, last_k):
  tf.reset_default_graph()

  checkpoint_state = tf.train.get_checkpoint_state(model_dir)
  checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
  var_list = tf.contrib.framework.list_variables(checkpoints[0])
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step"):
      var_values[name] = np.zeros(shape)
  for checkpoint in checkpoints:
    reader = tf.contrib.framework.load_checkpoint(checkpoint)
    for name in var_values:
      tensor = reader.get_tensor(name)
      var_dtypes[name] = tensor.dtype
      var_values[name] += tensor
    tf.logging.info("Read from checkpoint %s", checkpoint)
  for name in var_values:  # Average.
    var_values[name] /= len(checkpoints)

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})
    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
        global_step=global_step)
//  [python]
def avg_checkpoints(model_dir, output_model_dir, last_k):
  tf.reset_default_graph()

  checkpoint_state = tf.train.get_checkpoint_state(model_dir)
  checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
  var_list = tf.contrib.framework.list_variables(checkpoints[0])
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step"):
      var_values[name] = np.zeros(shape)
  for checkpoint in checkpoints:
    reader = tf.contrib.framework.load_checkpoint(checkpoint)
    for name in var_values:
      tensor = reader.get_tensor(name)
      var_dtypes[name] = tensor.dtype
      var_values[name] += tensor
    tf.logging.info("Read from checkpoint %s", checkpoint)
  for name in var_values:  # Average.
    var_values[name] /= len(checkpoints)

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.global_variables())

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})
    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
        global_step=global_step)
//  [python]
def make_prelu_tests(options):
  """Make a set of tests to do PReLU."""

  test_parameters = [
      {
          # The canonical case for image processing is having a 4D `input`
          # (NHWC)and `shared_axes`=[1, 2], so the alpha parameter is per
          # channel.
          "input_shape": [[1, 10, 10, 3], [3, 3, 3, 3]],
          "shared_axes": [[1, 2], [1]],
      },
      {
          # 2D-3D example. Share the 2nd axis.
          "input_shape": [[20, 20], [20, 20, 20]],
          "shared_axes": [[1]],
      }
  ]

  def build_graph(parameters):
    """Build the graph for the test case."""

    input_tensor = tf.placeholder(
        dtype=tf.float32, name="input", shape=parameters["input_shape"])
    prelu = tf.keras.layers.PReLU(shared_axes=parameters["shared_axes"])
    out = prelu(input_tensor)
    return [input_tensor], [out]

  def build_inputs(parameters, sess, inputs, outputs):
    """Build the inputs for the test case."""

    input_shape = parameters["input_shape"]
    input_values = create_tensor_data(
        np.float32, input_shape, min_value=-10, max_value=10)
    shared_axes = parameters["shared_axes"]

    alpha_shape = []
    for dim in range(1, len(input_shape)):
      alpha_shape.append(1 if dim in shared_axes else input_shape[dim])

    alpha_values = create_tensor_data(np.float32, alpha_shape)

    # There should be only 1 trainable variable tensor.
    variables = tf.all_variables()
    assert len(variables) == 1
    sess.run(variables[0].assign(alpha_values))

    return [input_values], sess.run(
        outputs, feed_dict=dict(zip(inputs, [input_values])))

  make_zip_of_tests(
      options,
      test_parameters,
      build_graph,
      build_inputs,
      use_frozen_graph=True)
//  [python]
def make_prelu_tests(options):
  """Make a set of tests to do PReLU."""

  test_parameters = [
      {
          # The canonical case for image processing is having a 4D `input`
          # (NHWC)and `shared_axes`=[1, 2], so the alpha parameter is per
          # channel.
          "input_shape": [[1, 10, 10, 3], [3, 3, 3, 3]],
          "shared_axes": [[1, 2], [1]],
      },
      {
          # 2D-3D example. Share the 2nd axis.
          "input_shape": [[20, 20], [20, 20, 20]],
          "shared_axes": [[1]],
      }
  ]

  def build_graph(parameters):
    """Build the graph for the test case."""

    input_tensor = tf.placeholder(
        dtype=tf.float32, name="input", shape=parameters["input_shape"])
    prelu = tf.keras.layers.PReLU(shared_axes=parameters["shared_axes"])
    out = prelu(input_tensor)
    return [input_tensor], [out]

  def build_inputs(parameters, sess, inputs, outputs):
    """Build the inputs for the test case."""

    input_shape = parameters["input_shape"]
    input_values = create_tensor_data(
        np.float32, input_shape, min_value=-10, max_value=10)
    shared_axes = parameters["shared_axes"]

    alpha_shape = []
    for dim in range(1, len(input_shape)):
      alpha_shape.append(1 if dim in shared_axes else input_shape[dim])

    alpha_values = create_tensor_data(np.float32, alpha_shape)

    # There should be only 1 trainable variable tensor.
    variables = tf.global_variables()
    assert len(variables) == 1
    sess.run(variables[0].assign(alpha_values))

    return [input_values], sess.run(
        outputs, feed_dict=dict(zip(inputs, [input_values])))

  make_zip_of_tests(
      options,
      test_parameters,
      build_graph,
      build_inputs,
      use_frozen_graph=True)
//  [python]
def print_all_variables(train_only=False):
    """Print all trainable and non-trainable variables
    without tl.layers.initialize_global_variables(sess)

    Parameters
    ----------
    train_only : boolean
        If True, only print the trainable variables, otherwise, print all variables.
    """
    # tvar = tf.trainable_variables() if train_only else tf.all_variables()
    if train_only:
        t_vars = tf.trainable_variables()
        print("  [*] printing trainable variables")
    else:
        try: # TF1.0
            t_vars = tf.global_variables()
        except: # TF0.12
            t_vars = tf.all_variables()
        print("  [*] printing global variables")
    for idx, v in enumerate(t_vars):
        print("  var {:3}: {:15}   {}".format(idx, str(v.get_shape()), v.name))
//  [python]
def print_all_variables(train_only=False):
    """Print all trainable and non-trainable variables
    without tl.layers.initialize_global_variables(sess)

    Parameters
    ----------
    train_only : boolean
        If True, only print the trainable variables, otherwise, print all variables.
    """
    if train_only:
        t_vars = tf.trainable_variables()
        print("  [*] printing trainable variables")
    else:
        t_vars = tf.global_variables()
        print("  [*] printing global variables")
    for idx, v in enumerate(t_vars):
        print("  var {:3}: {:15}   {}".format(idx, str(v.get_shape()), v.name))
//  [python]
def Give_TotalParameters():
    total_parameters = 0
    # iterating over all variables
    for variable in tf.all_variables():  # 统计全部的参数
        # for variable in tf.trainable_variables():#统计可以训练的参数
        local_parameters = 1
        shape = variable.get_shape()  # getting shape of a variable
        for i in shape:
            local_parameters *= i.value  # mutiplying dimension values
            total_parameters += local_parameters

    return total_parameters
//  [python]
def Give_TotalParameters():
    total_parameters = 0
    # iterating over all variables
    for variable in tf.global_variables():  # 统计全部的参数
        # for variable in tf.trainable_variables():#统计可以训练的参数
        local_parameters = 1
        shape = variable.get_shape()  # getting shape of a variable
        for i in shape:
            local_parameters *= i.value  # mutiplying dimension values
            total_parameters += local_parameters

    return total_parameters
//  [python]
def main(_):
  tf.logging.set_verbosity(tf.logging.INFO)

  reader = tf.train.load_checkpoint(FLAGS.src_ckpt)
  var_values, var_dtypes = {}, {}
  for (name, _) in tf.train.list_variables(FLAGS.src_ckpt):
    # skip global_step and optimizer states in src ckpt if not FLAGS.retain_all
    if FLAGS.finetune_only and (name.startswith("global_step") or
                                "adam" in name.lower() or
                                "generator" in name.lower()):
      continue

    tensor, tgt_name = get_tensor(reader, name)
    var_values[tgt_name] = tensor
    var_dtypes[tgt_name] = tensor.dtype

  if FLAGS.tgt_ckpt:
    if not tf.io.gfile.exists(os.path.dirname(FLAGS.tgt_ckpt)):
      tf.io.gfile.makedirs(os.path.dirname(FLAGS.tgt_ckpt))

    with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
      tf_vars = [
          tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
          for v in var_values
      ]
    placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
    assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
    group_assign_op = tf.group(assign_ops)
    saver = tf.train.Saver(tf.all_variables())

    with tf.Session() as sess:
      sess.run(tf.global_variables_initializer())

      # Run grouped assign op
      feed_dict = {}
      for p, (name, value) in zip(placeholders, var_values.items()):
        feed_dict[p] = value
      sess.run(group_assign_op, feed_dict)

      # Use the built saver to save the averaged checkpoint.
      saver.save(sess, FLAGS.tgt_ckpt)

    tf.logging.info("Modified checkpoint saved to %s", FLAGS.tgt_ckpt)
//  [python]
def main(_):
  tf.logging.set_verbosity(tf.logging.INFO)

  reader = tf.train.load_checkpoint(FLAGS.src_ckpt)
  var_values, var_dtypes = {}, {}
  for (name, _) in tf.train.list_variables(FLAGS.src_ckpt):
    # skip global_step and optimizer states in src ckpt if not FLAGS.retain_all
    if FLAGS.finetune_only and (name.startswith("global_step") or
                                "adam" in name.lower() or
                                "generator" in name.lower()):
      continue

    tensor, tgt_name = get_tensor(reader, name)
    var_values[tgt_name] = tensor
    var_dtypes[tgt_name] = tensor.dtype

  if FLAGS.tgt_ckpt:
    if not tf.io.gfile.exists(os.path.dirname(FLAGS.tgt_ckpt)):
      tf.io.gfile.makedirs(os.path.dirname(FLAGS.tgt_ckpt))

    with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
      tf_vars = [
          tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
          for v in var_values
      ]
    placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
    assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
    group_assign_op = tf.group(assign_ops)
    saver = tf.train.Saver(tf.global_variables())

    with tf.Session() as sess:
      sess.run(tf.global_variables_initializer())

      # Run grouped assign op
      feed_dict = {}
      for p, (name, value) in zip(placeholders, var_values.items()):
        feed_dict[p] = value
      sess.run(group_assign_op, feed_dict)

      # Use the built saver to save the averaged checkpoint.
      saver.save(sess, FLAGS.tgt_ckpt)

    tf.logging.info("Modified checkpoint saved to %s", FLAGS.tgt_ckpt)
//  [python]
def __new__(cls, *args, **kwargs):
    print(['Tensor.__new__', cls, args, kwargs])
    try:
      instance = super().__new__(cls, *args, **kwargs)
    except TypeError:
      instance = super().__new__(cls, **kwargs)
    if False and len(args) > 0 and isinstance(args[0], tf.Operation) and len(args[0].inputs) > 0:
      handle = args[0].inputs[0]
      if hasattr(handle, '_names'):
        import pdb; pdb.set_trace()
        instance._names = copy(handle._names)
        print('Set names', instance._names)
      else:
        vs = tf.all_variables()
        xs = [x.handle for x in vs if hasattr(x, 'handle')]
        if handle in xs:
          v = vs[xs.index(handle)]
          if hasattr(v, '_names'):
            instance._names = copy(v._names)
            print('Set names', instance._names)
      # print('TKTK')
      # setattr(instance, '_names', args[0]._names)
    return instance
//  [python]
def __new__(cls, *args, **kwargs):
    print(['Tensor.__new__', cls, args, kwargs])
    try:
      instance = super().__new__(cls, *args, **kwargs)
    except TypeError:
      instance = super().__new__(cls, **kwargs)
    if False and len(args) > 0 and isinstance(args[0], tf.Operation) and len(args[0].inputs) > 0:
      handle = args[0].inputs[0]
      if hasattr(handle, '_names'):
        import pdb; pdb.set_trace()
        instance._names = copy(handle._names)
        print('Set names', instance._names)
      else:
        vs = tf.global_variables()
        xs = [x.handle for x in vs if hasattr(x, 'handle')]
        if handle in xs:
          v = vs[xs.index(handle)]
          if hasattr(v, '_names'):
            instance._names = copy(v._names)
            print('Set names', instance._names)
      # print('TKTK')
      # setattr(instance, '_names', args[0]._names)
    return instance
//  [python]
def get_variables_with_name(name, train_only=True, printable=False):
    """Get variable list by a given name scope.

    Examples
    ---------
    >>> dense_vars = tl.layers.get_variable_with_name('dense', True, True)
    """
    print("  [*] geting variables with %s" % name)
    # tvar = tf.trainable_variables() if train_only else tf.all_variables()
    if train_only:
        t_vars = tf.trainable_variables()
    else:
        try: # TF1.0
            t_vars = tf.global_variables()
        except: # TF0.12
            t_vars = tf.all_variables()

    d_vars = [var for var in t_vars if name in var.name]
    if printable:
        for idx, v in enumerate(d_vars):
            print("  got {:3}: {:15}   {}".format(idx, v.name, str(v.get_shape())))
    return d_vars
//  [python]
def get_variables_with_name(name, train_only=True, printable=False):
    """Get variable list by a given name scope.

    Examples
    ---------
    >>> dense_vars = tl.layers.get_variable_with_name('dense', True, True)
    """
    print("  [*] geting variables with %s" % name)
    if train_only:
        t_vars = tf.trainable_variables()
    else:
        t_vars = tf.global_variables()

    d_vars = [var for var in t_vars if name in var.name]
    if printable:
        for idx, v in enumerate(d_vars):
            print("  got {:3}: {:15}   {}".format(idx, v.name, str(v.get_shape())))
    return d_vars