jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def make_restore_average_vars_dict():
  """Returns dict mapping moving average names to variables."""
  var_restore_dict = {}
  variable_averages = tf.train.ExponentialMovingAverage(0.999)
  for v in tf.global_variables():
    if v in tf.trainable_variables():
      name = variable_averages.average_name(v)
    else:
      name = v.op.name
    var_restore_dict[name] = v
  return var_restore_dict
//  [python]
def save_kernel_points(self, model, epoch):
        """
        Method saving kernel point disposition and current model weights for later visualization
        """

        if model.config.saving:

            # Create a directory to save kernels of this epoch
            kernels_dir = join(model.saving_path, 'kernel_points', 'epoch{:d}'.format(epoch))
            if not exists(kernels_dir):
                makedirs(kernels_dir)

            # Get points
            all_kernel_points_tf = [v for v in tf.compat.v1.global_variables() if 'kernel_points' in v.name
                                    and v.name.startswith('KernelPoint')]
            all_kernel_points = self.sess.run(all_kernel_points_tf)

            # Get Extents
            if False and 'gaussian' in model.config.convolution_mode:
                all_kernel_params_tf = [v for v in tf.compat.v1.global_variables() if 'kernel_extents' in v.name
                                        and v.name.startswith('KernelPoint')]
                all_kernel_params = self.sess.run(all_kernel_params_tf)
            else:
                all_kernel_params = [None for p in all_kernel_points]

            # Save in ply file
            for kernel_points, kernel_extents, v in zip(all_kernel_points, all_kernel_params, all_kernel_points_tf):

                # Name of saving file
                ply_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.ply'
                ply_file = join(kernels_dir, ply_name)

                # Data to save
                if kernel_points.ndim > 2:
                    kernel_points = kernel_points[:, 0, :]
                if False and 'gaussian' in model.config.convolution_mode:
                    data = [kernel_points, kernel_extents]
                    keys = ['x', 'y', 'z', 'sigma']
                else:
                    data = kernel_points
                    keys = ['x', 'y', 'z']

                # Save
                write_ply(ply_file, data, keys)

            # Get Weights
            all_kernel_weights_tf = [v for v in tf.compat.v1.global_variables() if 'weights' in v.name
                                    and v.name.startswith('KernelPointNetwork')]
            all_kernel_weights = self.sess.run(all_kernel_weights_tf)

            # Save in numpy file
            for kernel_weights, v in zip(all_kernel_weights, all_kernel_weights_tf):
                np_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.npy'
                np_file = join(kernels_dir, np_name)
                np.save(np_file, kernel_weights)
//  [python]
def save_kernel_points(self, model, epoch):
        """
        Method saving kernel point disposition and current model weights for later visualization
        """

        if model.config.saving:

            # Create a directory to save kernels of this epoch
            kernels_dir = join(model.saving_path, 'kernel_points', 'epoch{:d}'.format(epoch))
            if not exists(kernels_dir):
                makedirs(kernels_dir)

            # Get points
            all_kernel_points_tf = [v for v in tf.global_variables() if 'kernel_points' in v.name
                                    and v.name.startswith('KernelPoint')]
            all_kernel_points = self.sess.run(all_kernel_points_tf)

            # Get Extents
            if False and 'gaussian' in model.config.convolution_mode:
                all_kernel_params_tf = [v for v in tf.global_variables() if 'kernel_extents' in v.name
                                        and v.name.startswith('KernelPoint')]
                all_kernel_params = self.sess.run(all_kernel_params_tf)
            else:
                all_kernel_params = [None for p in all_kernel_points]

            # Save in ply file
            for kernel_points, kernel_extents, v in zip(all_kernel_points, all_kernel_params, all_kernel_points_tf):

                # Name of saving file
                ply_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.ply'
                ply_file = join(kernels_dir, ply_name)

                # Data to save
                if kernel_points.ndim > 2:
                    kernel_points = kernel_points[:, 0, :]
                if False and 'gaussian' in model.config.convolution_mode:
                    data = [kernel_points, kernel_extents]
                    keys = ['x', 'y', 'z', 'sigma']
                else:
                    data = kernel_points
                    keys = ['x', 'y', 'z']

                # Save
                write_ply(ply_file, data, keys)

            # Get Weights
            all_kernel_weights_tf = [v for v in tf.global_variables() if 'weights' in v.name
                                    and v.name.startswith('KernelPointNetwork')]
            all_kernel_weights = self.sess.run(all_kernel_weights_tf)

            # Save in numpy file
            for kernel_weights, v in zip(all_kernel_weights, all_kernel_weights_tf):
                np_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.npy'
                np_file = join(kernels_dir, np_name)
                np.save(np_file, kernel_weights)
//  [python]
def initialize_tf_variables(session, only_uninitialized=True):
    variables = tf.global_variables() + tf.local_variables()

    def is_initialized(variable):
        try:
            session.run(variable)
            return True
        except tf.errors.FailedPreconditionError:
            return False

        return False

    if only_uninitialized:
        variables = [
            variable for variable in variables
            if not is_initialized(variable)
        ]

    session.run(tf.variables_initializer(variables))
//  [python]
def initialize_uninitialized(sess):
	global_vars = tf.global_variables()
	is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
	not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
	# print('not_initialized_vars:')
	# for i in not_initialized_vars:
	# 	print(str(i.name))
	if len(not_initialized_vars):
		sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def get_vars(scope):
  return [x for x in tf.compat.v1.global_variables() if scope in x.name]
//  [python]
def get_vars(scope):
  return [x for x in tf.global_variables() if scope in x.name]
//  [python]
def adam_filter(model_path):
    """
    去掉模型中的Adam相关参数，这些参数在测试的时候是没有用的
    :param model_path: 
    :return: 
    """
    last_name = get_last_checkpoint(model_path)
    if last_name is None:
        return
    sess = tf.Session()
    imported_meta = tf.train.import_meta_graph(os.path.join(model_path, last_name + '.meta'))
    imported_meta.restore(sess, os.path.join(model_path, last_name))
    need_vars = []
    for var in tf.all_variables():
        if 'adam_v' not in var.name and 'adam_m' not in var.name:
            need_vars.append(var)
    saver = tf.train.Saver(need_vars)
    saver.save(sess, os.path.join(model_path, 'model.ckpt'))
//  [python]
def adam_filter(model_path):
    """
    去掉模型中的Adam相关参数，这些参数在测试的时候是没有用的
    :param model_path: 
    :return: 
    """
    last_name = get_last_checkpoint(model_path)
    if last_name is None:
        return
    sess = tf.Session()
    imported_meta = tf.train.import_meta_graph(os.path.join(model_path, last_name + '.meta'))
    imported_meta.restore(sess, os.path.join(model_path, last_name))
    need_vars = []
    for var in tf.global_variables():
        if 'adam_v' not in var.name and 'adam_m' not in var.name:
            need_vars.append(var)
    saver = tf.train.Saver(need_vars)
    saver.save(sess, os.path.join(model_path, 'model.ckpt'))
//  [python]
def __init__(self, layers, input_shape, scope=None):
        """ResNet constructor.

        :param layers: a list of layers in CleverHans format
          each with set_input_shape() and fprop() methods.
        :param input_shape: 4-tuple describing input shape (e.g None, 32, 32, 3)
        :param scope: string name of scope for Variables
          This works in two ways.
          If scope is None, the variables are not put in a scope, and the
          model is compatible with Saver.restore from the public downloads
          for the CIFAR10 Challenge.
          If the scope is a string, then Saver.restore won't work, but the
          model functions as a picklable NoRefModels that finds its variables
          based on the scope.
        """
        super(ResNet, self).__init__(scope, 10, {}, scope is not None)
        if scope is None:
            before = list(tf.trainable_variables())
            before_vars = list(tf.global_variables())
            self.build(layers, input_shape)
            after = list(tf.trainable_variables())
            after_vars = list(tf.global_variables())
            self.params = [param for param in after if param not in before]
            self.vars = [var for var in after_vars if var not in before_vars]
        else:
            with tf.variable_scope(self.scope):
                self.build(layers, input_shape)
//  [python]
def initialize_uninitialized(sess):
    """
    from https://stackoverflow.com/questions/35164529/in-tensorflow-is-there-any-way-to-just-initialize-uninitialised-variables
    """
    global_vars = tf.global_variables()
    is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def create_optimizer(self, config):
    momentum = config.float("momentum", 0.9)
    if self.opt_str == "sgd_nesterov":
      return tf.train.MomentumOptimizer(self.lr_var, momentum, use_nesterov=True), None
    elif self.opt_str == "sgd_momentum":
      return tf.train.MomentumOptimizer(self.lr_var, momentum), None
    elif self.opt_str == "sgd":
      return tf.train.GradientDescentOptimizer(self.lr_var), None
    elif self.opt_str == "adam":
      opt = tf.train.AdamOptimizer(self.lr_var)
      all_vars = tf.all_variables()
      opt_vars = [v for v in all_vars if "Adam" in v.name]
      reset_opt_op = tf.variables_initializer(opt_vars, "reset_optimizer")
      return opt, reset_opt_op
    elif self.opt_str == "none":
      return None, None
    else:
      assert False, ("unknown optimizer", self.opt_str)
//  [python]
def create_optimizer(self, config):
    momentum = config.float("momentum", 0.9)
    if self.opt_str == "sgd_nesterov":
      return tf.train.MomentumOptimizer(self.lr_var, momentum, use_nesterov=True), None
    elif self.opt_str == "sgd_momentum":
      return tf.train.MomentumOptimizer(self.lr_var, momentum), None
    elif self.opt_str == "sgd":
      return tf.train.GradientDescentOptimizer(self.lr_var), None
    elif self.opt_str == "adam":
      opt = tf.train.AdamOptimizer(self.lr_var)
      all_vars = tf.global_variables()
      opt_vars = [v for v in all_vars if "Adam" in v.name]
      reset_opt_op = tf.variables_initializer(opt_vars, "reset_optimizer")
      return opt, reset_opt_op
    elif self.opt_str == "none":
      return None, None
    else:
      assert False, ("unknown optimizer", self.opt_str)
//  [python]
def load_old_graph(self, ckpt): 
    ckpt_loader = create_loader(ckpt)
    self.say(old_graph_msg.format(ckpt))
    
    for var in tf.all_variables():
        name = var.name.split(':')[0]
        args = [name, var.get_shape()]
        val = ckpt_loader(args)
        assert val is not None, \
        'Cannot find and load {}'.format(var.name)
        shp = val.shape
        plh = tf.placeholder(tf.float32, shp)
        op = tf.assign(var, plh)
        self.sess.run(op, {plh: val})
//  [python]
def load_old_graph(self, ckpt): 
    ckpt_loader = create_loader(ckpt)
    self.say(old_graph_msg.format(ckpt))
    
    for var in tf.global_variables():
        name = var.name.split(':')[0]
        args = [name, var.get_shape()]
        val = ckpt_loader(args)
        assert val is not None, \
        'Cannot find and load {}'.format(var.name)
        shp = val.shape
        plh = tf.placeholder(tf.float32, shp)
        op = tf.assign(var, plh)
        self.sess.run(op, {plh: val})
//  [python]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):  # 这句话的意思是当运行下面的内容(train_op) 时，一定先执行 update_ops 的所有操作
    train_gen = optimizer_gen.minimize(gen_loss, var_list=gen_vars, global_step=global_step)
    train_disc = optimizer_disc.minimize(disc_loss, var_list=disc_vars)

# 模型要保存的变量
# var_list = tf.global_variables() + tf.local_variables()  # 这样保存所有变量不会出错，但是很多没必要的变量也保存了。414M 每个 ckpt
var_list = tf.trainable_variables()
if global_step not in var_list:
    var_list.append(global_step)
# 添加 BN 层的均值和方差
global_vars = tf.global_variables()
//  [python]
def restore_from_classification_checkpoint_fn(
      self,
      first_stage_feature_extractor_scope,
      second_stage_feature_extractor_scope):
    """Returns a map of variables to load from a foreign checkpoint.

    Note that this overrides the default implementation in
    faster_rcnn_meta_arch.FasterRCNNFeatureExtractor which does not work for
    NASNet-A checkpoints.

    Args:
      first_stage_feature_extractor_scope: A scope name for the first stage
        feature extractor.
      second_stage_feature_extractor_scope: A scope name for the second stage
        feature extractor.

    Returns:
      A dict mapping variable names (to load from a checkpoint) to variables in
      the model graph.
    """
    # Note that the NAS checkpoint only contains the moving average version of
    # the Variables so we need to generate an appropriate dictionary mapping.
    variables_to_restore = {}
    for variable in tf.global_variables():
      if variable.op.name.startswith(
          first_stage_feature_extractor_scope):
        var_name = variable.op.name.replace(
            first_stage_feature_extractor_scope + '/', '')
        var_name += '/ExponentialMovingAverage'
        variables_to_restore[var_name] = variable
      if variable.op.name.startswith(
          second_stage_feature_extractor_scope):
        var_name = variable.op.name.replace(
            second_stage_feature_extractor_scope + '/', '')
        var_name += '/ExponentialMovingAverage'
        variables_to_restore[var_name] = variable
    return variables_to_restore
//  [python]
def to_darknet(self):
    darknet_ckpt = self.darknet

    with self.graph.as_default() as g:
        for var in tf.global_variables():
            name = var.name.split(':')[0]
            var_name = name.split('-')
            l_idx = int(var_name[0])
            w_sig = var_name[1].split('/')[-1]
            l = darknet_ckpt.layers[l_idx]
            l.w[w_sig] = var.eval(self.sess)

    for layer in darknet_ckpt.layers:
        for ph in layer.h:
            layer.h[ph] = None

    return darknet_ckpt
//  [python]
def pickle_tf_vars():  
    """
    Saves tensorflow variables
    Requires them to be initialized first, also a default session must exist
    """
    _dict = {v.name : v.eval() for v in tf.all_variables()}
    with open(osp.join(G.output_dir, "vars.pkl"), 'wb') as f:
        pickle.dump(_dict, f)
//  [python]
def pickle_tf_vars():  
    """
    Saves tensorflow variables
    Requires them to be initialized first, also a default session must exist
    """
    _dict = {v.name : v.eval() for v in tf.global_variables()}
    with open(osp.join(G.output_dir, "vars.pkl"), 'wb') as f:
        pickle.dump(_dict, f)
//  [python]
def init_from_checkpoint(FLAGS, global_vars=False):
  tvars = tf.global_variables() if global_vars else tf.trainable_variables()
  initialized_variable_names = {}
  scaffold_fn = None
  if FLAGS.init_checkpoint is not None:
    if FLAGS.init_checkpoint.endswith("latest"):
      ckpt_dir = os.path.dirname(FLAGS.init_checkpoint)
      init_checkpoint = tf.train.latest_checkpoint(ckpt_dir)
    else:
      init_checkpoint = FLAGS.init_checkpoint

    tf.logging.info("Initialize from the ckpt {}".format(init_checkpoint))

    (assignment_map, initialized_variable_names
    ) = get_assignment_map_from_checkpoint(tvars, init_checkpoint)
    if FLAGS.use_tpu:
      def tpu_scaffold():
        tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
        return tf.train.Scaffold()

      scaffold_fn = tpu_scaffold
    else:
      tf.train.init_from_checkpoint(init_checkpoint, assignment_map)

    # Log customized initialization
    tf.logging.info("**** Global Variables ****")
    for var in tvars:
      init_string = ""
      if var.name in initialized_variable_names:
        init_string = ", *INIT_FROM_CKPT*"
      tf.logging.info("  name = %s, shape = %s%s", var.name, var.shape,
                      init_string)
  return scaffold_fn
//  [python]
def test_export_model_with_quantization_nodes(self):
    tmp_dir = self.get_temp_dir()
    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
    self._save_checkpoint_from_mock_model(
        trained_checkpoint_prefix,
        use_moving_averages=False,
        enable_quantization=True)
    output_directory = os.path.join(tmp_dir, 'output')
    inference_graph_path = os.path.join(output_directory,
                                        'inference_graph.pbtxt')
    with mock.patch.object(
        model_builder, 'build', autospec=True) as mock_builder:
      mock_builder.return_value = FakeModel()
      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
      text_format.Merge(
          """graph_rewriter {
               quantization {
                 delay: 50000
                 activation_bits: 8
                 weight_bits: 8
               }
             }""", pipeline_config)
      exporter.export_inference_graph(
          input_type='image_tensor',
          pipeline_config=pipeline_config,
          trained_checkpoint_prefix=trained_checkpoint_prefix,
          output_directory=output_directory,
          write_inference_graph=True)
    self._load_inference_graph(inference_graph_path, is_binary=False)
    has_quant_nodes = False
    for v in tf.compat.v1.global_variables():
      if v.op.name.endswith('act_quant/min'):
        has_quant_nodes = True
        break
    self.assertTrue(has_quant_nodes)
//  [python]
def test_export_model_with_quantization_nodes(self):
    tmp_dir = self.get_temp_dir()
    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
    self._save_checkpoint_from_mock_model(
        trained_checkpoint_prefix,
        use_moving_averages=False,
        enable_quantization=True)
    output_directory = os.path.join(tmp_dir, 'output')
    inference_graph_path = os.path.join(output_directory,
                                        'inference_graph.pbtxt')
    with mock.patch.object(
        model_builder, 'build', autospec=True) as mock_builder:
      mock_builder.return_value = FakeModel()
      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
      text_format.Merge(
          """graph_rewriter {
               quantization {
                 delay: 50000
                 activation_bits: 8
                 weight_bits: 8
               }
             }""", pipeline_config)
      exporter.export_inference_graph(
          input_type='image_tensor',
          pipeline_config=pipeline_config,
          trained_checkpoint_prefix=trained_checkpoint_prefix,
          output_directory=output_directory,
          write_inference_graph=True)
    self._load_inference_graph(inference_graph_path, is_binary=False)
    has_quant_nodes = False
    for v in tf.global_variables():
      if v.op.name.endswith('act_quant/min'):
        has_quant_nodes = True
        break
    self.assertTrue(has_quant_nodes)
//  [python]
def main(argv=None):
    if FLAGS.tiny:
        model = yolo_v3_tiny.yolo_v3_tiny
    elif FLAGS.spp:
        model = yolo_v3.yolo_v3_spp
    else:
        model = yolo_v3.yolo_v3

    classes = load_coco_names(FLAGS.class_names)

    # placeholder for detector inputs
    inputs = tf.placeholder(tf.float32, [None, FLAGS.size, FLAGS.size, 3], "inputs")

    with tf.variable_scope('detector'):
        detections = model(inputs, len(classes), data_format=FLAGS.data_format)
        load_ops = load_weights(tf.global_variables(scope='detector'), FLAGS.weights_file)

    # Sets the output nodes in the current session
    boxes = detections_boxes(detections)

    with tf.Session() as sess:
        sess.run(load_ops)
        freeze_graph(sess, FLAGS.output_graph)
//  [python]
def get_global_variable_by_name(name):
	"""Returns the global variable of given name

	name : the name of the global variable
	"""
	# return [v for v in tf.variables() if v.name == name][0]
	return [v for v in tf.global_variables() if v.name == name][0]
//  [python]
def inference_for_keypoints(image_dir, image_list, checkpoint, output_dir):

    # Create directory for output.
    if not os.path.exists(output_dir):
        os.system('mkdir ' + output_dir)

    # Create queue coordinator.
    coord = tf.train.Coordinator()

    # Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReader(
            image_dir,
            image_list,
            None,  # No defined input size.
            False,  # No random scale.
            False,  # No random mirror.
            IGNORE_LABEL,
            IMG_MEAN,
            coord)
        image, label = reader.image, reader.label

    # Add one batch dimension.
    image_batch, label_batch = tf.expand_dims(
        image, dim=0), tf.expand_dims(label, dim=0)

    # Create network.
    net = DeepLabResNetModel({'data': image_batch},
                             is_training=False, num_classes=NUM_CLASSES)
    restore_var = tf.global_variables()

    # Predictions.
    raw_output = net.layers['fc1_voc12']
    raw_output = tf.image.resize_bilinear(
        raw_output, tf.shape(image_batch)[1:3, ])
    raw_output = tf.argmax(raw_output, dimension=3)
    pred = tf.expand_dims(raw_output, dim=3)  # Create 4-d tensor.

    # Set up tf session and initialize variables.
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()

    sess.run(init)
    sess.run(tf.local_variables_initializer())

    # Load weights.
    tf.train.Saver(var_list=restore_var).restore(sess, checkpoint)
    print('Successfully restored model parameters from {}'.format(checkpoint))

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Read list text file.
    with open(image_list, 'r') as txtfile:
        files = txtfile.readlines()

    # Iterate for all images.
    start = time.time()
    for idx, file in enumerate(files):

        # Print status.
        if (idx + 1) % 1000 == 0 or (idx + 1) == len(files):
            print(str(idx + 1) + ' / ' + str(len(files)))

        # Inference for human keypoints, save the label image.
        preds = sess.run(pred)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            io.imsave(output_dir + '/' + file.split('.')
                      [0] + '.png', np.uint8(np.squeeze(preds)))

    print('Successfully processed all the images in %.2f seconds.' %
          (time.time() - start))

    coord.request_stop()
    coord.join(threads)
//  [python]
def restore_into_scope(model_path, scope_name, sess):
    global_vars = tf.all_variables()
    tensors_to_load = [v for v in global_vars if v.name.startswith(scope_name + '/')]

    load_dict = {}
    for j in range(0, np.size(tensors_to_load)):
        tensor_name = tensors_to_load[j].name
        tensor_name = tensor_name[0:-2] # remove ':0'
        tensor_name = tensor_name.replace(scope_name + '/', '') #remove scope
        load_dict.update({tensor_name: tensors_to_load[j]})
    loader = tf.train.Saver(var_list=load_dict)
    loader.restore(sess, model_path)
    print("Model restored from: {0} into scope: {1}.".format(model_path, scope_name))
//  [python]
def restore_into_scope(model_path, scope_name, sess):
    global_vars = tf.global_variables()
    tensors_to_load = [v for v in global_vars if v.name.startswith(scope_name + '/')]

    load_dict = {}
    for j in range(0, np.size(tensors_to_load)):
        tensor_name = tensors_to_load[j].name
        tensor_name = tensor_name[0:-2] # remove ':0'
        tensor_name = tensor_name.replace(scope_name + '/', '') #remove scope
        load_dict.update({tensor_name: tensors_to_load[j]})
    loader = tf.train.Saver(var_list=load_dict)
    loader.restore(sess, model_path)
    print("Model restored from: {0} into scope: {1}.".format(model_path, scope_name))
//  [python]
def main(argv=None):
    if FLAGS.tiny:
        model = yolo_v3_tiny.yolo_v3_tiny
    elif FLAGS.spp:
        model = yolo_v3.yolo_v3_spp
    else:
        model = yolo_v3.yolo_v3

    classes = load_coco_names(FLAGS.class_names)

    # placeholder for detector inputs
    # any size > 320 will work here
    inputs = tf.placeholder(tf.float32, [None, 416, 416, 3])

    with tf.variable_scope('detector'):
        detections = model(inputs, len(classes),
                           data_format=FLAGS.data_format)
        load_ops = load_weights(tf.global_variables(
            scope='detector'), FLAGS.weights_file)

    saver = tf.train.Saver(tf.global_variables(scope='detector'))

    with tf.Session() as sess:
        sess.run(load_ops)

        save_path = saver.save(sess, save_path=FLAGS.ckpt_file)
        print('Model saved in path: {}'.format(save_path))
//  [python]
def main():

    success = True
    for i in range(10):
        w = np.random.randn(32, 16, 7, 7).astype(np.float32)
        x = np.random.randn(10, 128, 224, 224).astype(np.float32)

        gl_model = GluonModel()

        # ctx = mx.cpu()
        ctx = mx.gpu(0)
        gl_params = gl_model._collect_params_with_prefix()
        gl_params['g_conv.weight']._load_init(mx.nd.array(w, ctx), ctx)

        gl_x = mx.nd.array(x, ctx)
        gl_y = gl_model(gl_x).asnumpy()

        xx = tf.placeholder(
            dtype=tf.float32,
            shape=(None, 128, 224, 224),
            name='xx')
        tf_model = tensorflow_model(xx)
        tf_params = {v.name: v for v in tf.all_variables()}
        with tf.Session() as sess:
            w_list = np.split(w, axis=0, indices_or_sections=GROUPS)
            for gi in range(GROUPS):
                w_gi = w_list[gi]
                tf_w = np.transpose(w_gi, axes=(2, 3, 1, 0))
                sess.run(tf_params['g_conv/convgroup{}/kernel:0'.format(gi + 1)].assign(tf_w))

            tf_y = sess.run(tf_model, feed_dict={xx: x})
        tf.reset_default_graph()

        dist = np.sum(np.abs(gl_y - tf_y))
        if dist > 1e-5:
            success = False
            print("i={}, dist={}".format(i, dist))
            # print(gl_y)
            # print(tf_y)

    if success:
        print("All ok.")
//  [python]
def main():

    success = True
    for i in range(10):
        w = np.random.randn(32, 16, 7, 7).astype(np.float32)
        x = np.random.randn(10, 128, 224, 224).astype(np.float32)

        gl_model = GluonModel()

        # ctx = mx.cpu()
        ctx = mx.gpu(0)
        gl_params = gl_model._collect_params_with_prefix()
        gl_params['g_conv.weight']._load_init(mx.nd.array(w, ctx), ctx)

        gl_x = mx.nd.array(x, ctx)
        gl_y = gl_model(gl_x).asnumpy()

        xx = tf.placeholder(
            dtype=tf.float32,
            shape=(None, 128, 224, 224),
            name='xx')
        tf_model = tensorflow_model(xx)
        tf_params = {v.name: v for v in tf.global_variables()}
        with tf.Session() as sess:
            w_list = np.split(w, axis=0, indices_or_sections=GROUPS)
            for gi in range(GROUPS):
                w_gi = w_list[gi]
                tf_w = np.transpose(w_gi, axes=(2, 3, 1, 0))
                sess.run(tf_params['g_conv/convgroup{}/kernel:0'.format(gi + 1)].assign(tf_w))

            tf_y = sess.run(tf_model, feed_dict={xx: x})
        tf.reset_default_graph()

        dist = np.sum(np.abs(gl_y - tf_y))
        if dist > 1e-5:
            success = False
            print("i={}, dist={}".format(i, dist))
            # print(gl_y)
            # print(tf_y)

    if success:
        print("All ok.")
//  [python]
def main():

    success = True
    for i in range(10):
        # gl_w = np.random.randn(1000, 1024).astype(np.float32)
        tf_w = np.random.randn(1024, 1000).astype(np.float32)
        # b = np.random.randn(1000, ).astype(np.float32)
        x = np.random.randn(1, 1024).astype(np.float32)

        gl_model = GluonModel()

        # ctx = mx.cpu()
        ctx = mx.gpu(0)
        gl_params = gl_model._collect_params_with_prefix()
        gl_w = np.transpose(tf_w, axes=(1, 0))
        gl_params['dense.weight']._load_init(mx.nd.array(gl_w, ctx), ctx)
        # gl_params['dense.bias']._load_init(mx.nd.array(b, ctx), ctx)

        gl_x = mx.nd.array(x, ctx)
        gl_y = gl_model(gl_x).asnumpy()

        xx = tf.placeholder(
            dtype=tf.float32,
            shape=(None, 1024),
            name='xx')
        tf_model = tensorflow_model(xx)
        tf_params = {v.name: v for v in tf.all_variables()}
        with tf.Session() as sess:
            # tf_w = np.transpose(gl_w, axes=(1, 0))
            sess.run(tf_params['dense/kernel:0'].assign(tf_w))
            # sess.run(tf_params['dense/bias:0'].assign(b))
            # sess.run(tf_params['dense/weights:0'].assign(tf_w))
            # sess.run(tf_params['dense/biases:0'].assign(b))

            tf_y = sess.run(tf_model, feed_dict={xx: x})
        tf.reset_default_graph()

        dist = np.sum(np.abs(gl_y - tf_y))
        if dist > 1e-5:
            success = False
            print("i={}, dist={}".format(i, dist))
            # print(gl_y)
            # print(tf_y)
            y = np.matmul(gl_w.astype(np.float64), x[0].astype(np.float64))
            # y = np.dot(w, x[0])
            gl_dist = np.sum(np.abs(gl_y - y))
            tf_dist = np.sum(np.abs(tf_y - y))
            print("i={}, gl_dist={}".format(i, gl_dist))
            print("i={}, tf_dist={}".format(i, tf_dist))

    if success:
        print("All ok.")
//  [python]
def main():

    success = True
    for i in range(10):
        # gl_w = np.random.randn(1000, 1024).astype(np.float32)
        tf_w = np.random.randn(1024, 1000).astype(np.float32)
        # b = np.random.randn(1000, ).astype(np.float32)
        x = np.random.randn(1, 1024).astype(np.float32)

        gl_model = GluonModel()

        # ctx = mx.cpu()
        ctx = mx.gpu(0)
        gl_params = gl_model._collect_params_with_prefix()
        gl_w = np.transpose(tf_w, axes=(1, 0))
        gl_params['dense.weight']._load_init(mx.nd.array(gl_w, ctx), ctx)
        # gl_params['dense.bias']._load_init(mx.nd.array(b, ctx), ctx)

        gl_x = mx.nd.array(x, ctx)
        gl_y = gl_model(gl_x).asnumpy()

        xx = tf.placeholder(
            dtype=tf.float32,
            shape=(None, 1024),
            name='xx')
        tf_model = tensorflow_model(xx)
        tf_params = {v.name: v for v in tf.global_variables()}
        with tf.Session() as sess:
            # tf_w = np.transpose(gl_w, axes=(1, 0))
            sess.run(tf_params['dense/kernel:0'].assign(tf_w))
            # sess.run(tf_params['dense/bias:0'].assign(b))
            # sess.run(tf_params['dense/weights:0'].assign(tf_w))
            # sess.run(tf_params['dense/biases:0'].assign(b))

            tf_y = sess.run(tf_model, feed_dict={xx: x})
        tf.reset_default_graph()

        dist = np.sum(np.abs(gl_y - tf_y))
        if dist > 1e-5:
            success = False
            print("i={}, dist={}".format(i, dist))
            # print(gl_y)
            # print(tf_y)
            y = np.matmul(gl_w.astype(np.float64), x[0].astype(np.float64))
            # y = np.dot(w, x[0])
            gl_dist = np.sum(np.abs(gl_y - y))
            tf_dist = np.sum(np.abs(tf_y - y))
            print("i={}, gl_dist={}".format(i, gl_dist))
            print("i={}, tf_dist={}".format(i, tf_dist))

    if success:
        print("All ok.")
//  [python]
def get_all_variable():
  var_list = tf.trainable_variables() + tf.get_collection('moving_vars')
  for v in tf.global_variables():
    # We maintain ema for batch norm moving mean and variance as well.
    if 'moving_mean' in v.name or 'moving_variance' in v.name:
      var_list.append(v)
  var_list = list(set(var_list))
  var_list = sorted(var_list, key=lambda var: var.name)
  return var_list
//  [python]
def initialize_uninitialized(sess):
    global_vars          = tf.global_variables()
    is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

    print([str(i.name) for i in not_initialized_vars]) # only for testing
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def define_saver(exclude=None):
  """Create a saver for the variables we want to checkpoint.

  Args:
    exclude: List of regexes to match variable names to exclude.

  Returns:
    Saver object.
  """
  variables = []
  exclude = exclude or []
  exclude = [re.compile(regex) for regex in exclude]
  for variable in tf.all_variables():
    if any(regex.match(variable.name) for regex in exclude):
      continue
    variables.append(variable)
  saver = tf.train.Saver(variables, keep_checkpoint_every_n_hours=5)
  return saver
//  [python]
def define_saver(exclude=None):
  """Create a saver for the variables we want to checkpoint.

  Args:
    exclude: List of regexes to match variable names to exclude.

  Returns:
    Saver object.
  """
  variables = []
  exclude = exclude or []
  exclude = [re.compile(regex) for regex in exclude]
  for variable in tf.global_variables():
    if any(regex.match(variable.name) for regex in exclude):
      continue
    variables.append(variable)
  saver = tf.train.Saver(variables, keep_checkpoint_every_n_hours=5)
  return saver
//  [python]
def init_saver(self):
        """
        初始化saver对象
        :return:
        """
        self.saver = tf.train.Saver(tf.all_variables(), max_to_keep=100)
//  [python]
def init_saver(self):
        """
        初始化saver对象
        :return:
        """
        self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
//  [python]
def predict(words):
  word_data = reader.sentence_to_word_ids(FLAGS.pos_data_path, words)
  tag_data = [0]*len(word_data)

  config = get_config(FLAGS.pos_lang)
  config.batch_size = 1
  # config.num_steps, num_steps is T, should not be set to 1, CRF decode whe whole sequence at one time

  # intialize model
  ckpt_path = os.path.join(FLAGS.pos_train_dir, "pos_bilstm_crf.ckpt")
  
  with tf.Graph().as_default(), tf.Session() as session:
    with tf.variable_scope("pos_var_scope"):  #Need to Change in Pos_Tagger Save Function
      model = POSTagger(is_training=False, config=config) # save object after is_training
    
    if len(glob.glob(ckpt_path + '.data*')) > 0: # file exist with pattern: 'pos.ckpt.data*'
      print("Loading model parameters from %s" % ckpt_path)
      all_vars = tf.all_variables()
      model_vars = [k for k in all_vars if k.name.startswith("pos_var_scope")]
      tf.train.Saver(model_vars).restore(session, ckpt_path)
    else:
      print("Model not found, created with fresh parameters.")
      session.run(tf.initialize_all_variables())

    # Make Prediciton
    word_data = np.array(word_data, dtype=np.int32)
    sequence_len = len(word_data)
    x = np.zeros([1, config.num_steps], dtype=np.int32)
    for i in range(sequence_len):   # pad word_data len to num_steps
      x[0, i] = word_data[i]
    y = np.zeros([1, config.num_steps], dtype=np.int32)

    fetches = [model.cost, model.logits, model.transition_params]
    feed_dict = {}
    feed_dict[model.input_data] = x
    feed_dict[model.targets] = y
    cost, logits, transition_params = session.run(fetches, feed_dict)
    print (logits[0])
    print (logits)

    # print (transition_params)
    print (transition_params[0, :])

    # Remove padding from the scores and tag sequence.
    tf_unary_scores = logits[0, :, :]  # One Example [batch_size, num_steps, target_num]
    print ("unary_score %d" % len(tf_unary_scores))

    tf_unary_scores = tf_unary_scores[:sequence_len]
    print ("unary_score %d" % len(tf_unary_scores))

    # transition_params  [Aij] tag transition probability
    viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(tf_unary_scores, transition_params)
    print ("optimal sequence found by viterbi algorithm")
    print (viterbi_sequence)
    predict_tag = reader.word_ids_to_sentence(FLAGS.pos_data_path, viterbi_sequence)
  return predict_tag
//  [python]
def predict(words):
  word_data = reader.sentence_to_word_ids(FLAGS.pos_data_path, words)
  tag_data = [0]*len(word_data)

  config = get_config(FLAGS.pos_lang)
  config.batch_size = 1
  # config.num_steps, num_steps is T, should not be set to 1, CRF decode whe whole sequence at one time

  # intialize model
  ckpt_path = os.path.join(FLAGS.pos_train_dir, "pos_bilstm_crf.ckpt")
  
  with tf.Graph().as_default(), tf.Session() as session:
    with tf.variable_scope("pos_var_scope"):  #Need to Change in Pos_Tagger Save Function
      model = POSTagger(is_training=False, config=config) # save object after is_training
    
    if len(glob.glob(ckpt_path + '.data*')) > 0: # file exist with pattern: 'pos.ckpt.data*'
      print("Loading model parameters from %s" % ckpt_path)
      all_vars = tf.global_variables()
      model_vars = [k for k in all_vars if k.name.startswith("pos_var_scope")]
      tf.train.Saver(model_vars).restore(session, ckpt_path)
    else:
      print("Model not found, created with fresh parameters.")
      session.run(tf.global_variables_initializer())

    # Make Prediciton
    word_data = np.array(word_data, dtype=np.int32)
    sequence_len = len(word_data)
    x = np.zeros([1, config.num_steps], dtype=np.int32)
    for i in range(sequence_len):   # pad word_data len to num_steps
      x[0, i] = word_data[i]
    y = np.zeros([1, config.num_steps], dtype=np.int32)

    fetches = [model.cost, model.logits, model.transition_params]
    feed_dict = {}
    feed_dict[model.input_data] = x
    feed_dict[model.targets] = y
    cost, logits, transition_params = session.run(fetches, feed_dict)
    print (logits[0])
    print (logits)

    # print (transition_params)
    print (transition_params[0, :])

    # Remove padding from the scores and tag sequence.
    tf_unary_scores = logits[0, :, :]  # One Example [batch_size, num_steps, target_num]
    print ("unary_score %d" % len(tf_unary_scores))

    tf_unary_scores = tf_unary_scores[:sequence_len]
    print ("unary_score %d" % len(tf_unary_scores))

    # transition_params  [Aij] tag transition probability
    viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(tf_unary_scores, transition_params)
    print ("optimal sequence found by viterbi algorithm")
    print (viterbi_sequence)
    predict_tag = reader.word_ids_to_sentence(FLAGS.pos_data_path, viterbi_sequence)
  return predict_tag
//  [python]
def testNoBatchNormScaleByDefault(self):
    height, width = 299, 299
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with tf.contrib.slim.arg_scope(inception.inception_v4_arg_scope()):
      inception.inception_v4(inputs, num_classes, is_training=False)

    self.assertEqual(tf.global_variables('.*/BatchNorm/gamma:0$'), [])
//  [python]
def testBatchNormScale(self):
    height, width = 299, 299
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with tf.contrib.slim.arg_scope(
        inception.inception_v4_arg_scope(batch_norm_scale=True)):
      inception.inception_v4(inputs, num_classes, is_training=False)

    gamma_names = set(
        v.op.name for v in tf.global_variables('.*/BatchNorm/gamma:0$'))
    self.assertGreater(len(gamma_names), 0)
    for v in tf.global_variables('.*/BatchNorm/moving_mean:0$'):
      self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names)
//  [python]
def optimistic_restore(session, save_file):
    """
    Restores as many weigts as possible from a checkpoint filed, in contrast to tf
    function why returns an exeption if some weigts in the graph are not found in the checkpoint.
    Very convenient when you want to load pretrained part of your graph.
    
    session: session to restore the weigts to
    save_file: the checkpoint file
    
    returns the number of variables restored
    """
    reader = tf.train.NewCheckpointReader(save_file)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted([(var.name, var.name.split(':')[0]) for var in tf.all_variables() if var.name.split(':')[0] in saved_shapes])
    restore_vars = []
    name2var = dict(zip(map(lambda x:x.name.split(':')[0], tf.all_variables()), tf.all_variables()))
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = name2var[saved_var_name]
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
    saver = tf.train.Saver(restore_vars)
    saver.restore(session, save_file)
    return len(restore_vars)
//  [python]
def optimistic_restore(session, save_file):
    """
    Restores as many weigts as possible from a checkpoint filed, in contrast to tf
    function why returns an exeption if some weigts in the graph are not found in the checkpoint.
    Very convenient when you want to load pretrained part of your graph.
    
    session: session to restore the weigts to
    save_file: the checkpoint file
    
    returns the number of variables restored
    """
    reader = tf.train.NewCheckpointReader(save_file)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted([(var.name, var.name.split(':')[0]) for var in tf.global_variables()if var.name.split(':')[0] in saved_shapes])
    restore_vars = []
    name2var = dict(zip(map(lambda x:x.name.split(':')[0], tf.global_variables()), tf.global_variables()))
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = name2var[saved_var_name]
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
    saver = tf.train.Saver(restore_vars)
    saver.restore(session, save_file)
    return len(restore_vars)
//  [python]
def _get_saver_(self):
        # create a saver to save all variables
        # Saver op should always be assigned to cpu, and it should be
        # created after all variables have been defined; otherwise, it
        # only save those variables already created.
        with tf.device('/cpu:0'):
            if self.ckpt_var_list is None:
                return tf.train.Saver(var_list=tf.all_variables(), max_to_keep=2)
            else:
                return tf.train.Saver(var_list=self.ckpt_var_list, max_to_keep=2)
//  [python]
def _get_saver_(self):
        # create a saver to save all variables
        # Saver op should always be assigned to cpu, and it should be
        # created after all variables have been defined; otherwise, it
        # only save those variables already created.
        with tf.device('/cpu:0'):
            if self.ckpt_var_list is None:
                return tf.train.Saver(var_list=tf.global_variables(), max_to_keep=2)
            else:
                return tf.train.Saver(var_list=self.ckpt_var_list, max_to_keep=2)
//  [python]
def get_session():
    """Returns the TF session to be used by the backend.

    If a default TensorFlow session is available, we will return it.

    Else, we will return the global Keras session.

    If no global Keras session exists at this point:
    we will create a new global session.

    Note that you can manually set the global session
    via `K.set_session(sess)`.

    # Returns
        A TensorFlow session.
    """
    global _SESSION
    if tf.get_default_session() is not None:
        session = tf.get_default_session()
    else:
        if _SESSION is None:
            if not os.environ.get('OMP_NUM_THREADS'):
                config = tf.ConfigProto(allow_soft_placement=True)
            else:
                num_thread = int(os.environ.get('OMP_NUM_THREADS'))
                config = tf.ConfigProto(intra_op_parallelism_threads=num_thread,
                                        allow_soft_placement=True)
            _SESSION = tf.Session(config=config)
        session = _SESSION
    if not _MANUAL_VAR_INIT:
        with session.graph.as_default():
            variables = tf.all_variables()
            candidate_vars = []
            for v in variables:
                if not getattr(v, '_keras_initialized', False):
                    candidate_vars.append(v)
            if candidate_vars:
                # This step is expensive, so we only run it on variables
                # not already marked as initialized.
                is_initialized = session.run(
                    [tf.is_variable_initialized(v) for v in candidate_vars])
                uninitialized_vars = []
                for flag, v in zip(is_initialized, candidate_vars):
                    if not flag:
                        uninitialized_vars.append(v)
                    v._keras_initialized = True
                if uninitialized_vars:
                    session.run(tf.variables_initializer(uninitialized_vars))
    # hack for list_devices() function.
    # list_devices() function is not available under tensorflow r1.3.
    if not hasattr(session, 'list_devices'):
        session.list_devices = lambda: device_lib.list_local_devices()
    return session
//  [python]
def get_session():
    """Returns the TF session to be used by the backend.

    If a default TensorFlow session is available, we will return it.

    Else, we will return the global Keras session.

    If no global Keras session exists at this point:
    we will create a new global session.

    Note that you can manually set the global session
    via `K.set_session(sess)`.

    # Returns
        A TensorFlow session.
    """
    global _SESSION
    if tf.get_default_session() is not None:
        session = tf.get_default_session()
    else:
        if _SESSION is None:
            if not os.environ.get('OMP_NUM_THREADS'):
                config = tf.ConfigProto(allow_soft_placement=True)
            else:
                num_thread = int(os.environ.get('OMP_NUM_THREADS'))
                config = tf.ConfigProto(intra_op_parallelism_threads=num_thread,
                                        allow_soft_placement=True)
            _SESSION = tf.Session(config=config)
        session = _SESSION
    if not _MANUAL_VAR_INIT:
        with session.graph.as_default():
            variables = tf.global_variables()
            candidate_vars = []
            for v in variables:
                if not getattr(v, '_keras_initialized', False):
                    candidate_vars.append(v)
            if candidate_vars:
                # This step is expensive, so we only run it on variables
                # not already marked as initialized.
                is_initialized = session.run(
                    [tf.is_variable_initialized(v) for v in candidate_vars])
                uninitialized_vars = []
                for flag, v in zip(is_initialized, candidate_vars):
                    if not flag:
                        uninitialized_vars.append(v)
                    v._keras_initialized = True
                if uninitialized_vars:
                    session.run(tf.variables_initializer(uninitialized_vars))
    # hack for list_devices() function.
    # list_devices() function is not available under tensorflow r1.3.
    if not hasattr(session, 'list_devices'):
        session.list_devices = lambda: device_lib.list_local_devices()
    return session