jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def save_model(self, session, step):
        """
        保存模型
        """
        if not self.saver:
            var_list = tf.all_variables()

            """
            # 对于BN的保存改造
            g_list = tf.global_variables()
            bn_moving_vars = [g for g in g_list if "moving_mean" in g.name]
            bn_moving_vars += [g for g in g_list if "moving_variance" in g.name]

            if bn_moving_vars or bn_moving_vars:
                print("find BN variables!")
            var_list += bn_moving_vars
            """

            self.saver = tf.train.Saver(var_list=var_list, max_to_keep=100)

            vocab_file = os.path.join(self.args.bert_init_dir, self.args.vocab_file)
            des_vocab_file = os.path.join(self.args.model_save_dir, self.args.vocab_file)
            config_file = os.path.join(self.args.bert_init_dir, self.args.bert_config_file)
            des_config_file = os.path.join(self.args.model_save_dir, self.args.bert_config_file)

            # 如果是从原始BERT model开始保存的话，需要copy vocab和config过去
            if self.args.bert_init_dir != self.args.model_save_dir:
                shutil.copy(vocab_file, des_vocab_file)
                shutil.copy(config_file, des_config_file)

        model_file = os.path.join(self.args.model_save_dir, self.args.model_name + ".ckpt." + str(step))
        self.saver.save(session, model_file)
        return True
//  [python]
def save_model(self, session, step):
        """
        保存模型
        """
        if not self.saver:
            var_list = tf.global_variables()

            """
            # 对于BN的保存改造
            g_list = tf.global_variables()
            bn_moving_vars = [g for g in g_list if "moving_mean" in g.name]
            bn_moving_vars += [g for g in g_list if "moving_variance" in g.name]

            if bn_moving_vars or bn_moving_vars:
                print("find BN variables!")
            var_list += bn_moving_vars
            """

            self.saver = tf.train.Saver(var_list=var_list, max_to_keep=100)

            vocab_file = os.path.join(self.args.bert_init_dir, self.args.vocab_file)
            des_vocab_file = os.path.join(self.args.model_save_dir, self.args.vocab_file)
            config_file = os.path.join(self.args.bert_init_dir, self.args.bert_config_file)
            des_config_file = os.path.join(self.args.model_save_dir, self.args.bert_config_file)

            # 如果是从原始BERT model开始保存的话，需要copy vocab和config过去
            if self.args.bert_init_dir != self.args.model_save_dir:
                shutil.copy(vocab_file, des_vocab_file)
                shutil.copy(config_file, des_config_file)

        model_file = os.path.join(self.args.model_save_dir, self.args.model_name + ".ckpt." + str(step))
        self.saver.save(session, model_file)
        return True
//  [python]
def testNoneUpdatesCollectionsNHWC(self):
    self._testNoneUpdatesCollections(False, data_format='NHWC')
    print(tf.all_variables())
//  [python]
def testNoneUpdatesCollectionsNHWC(self):
    self._testNoneUpdatesCollections(False, data_format='NHWC')
    print(tf.global_variables())
//  [python]
def get_train_op(self, total_loss, global_step, learning_rate):
        """
        Return train_op
        """
        optimizer = tf.train.AdamOptimizer(learning_rate)
        train_op = optimizer.minimize(total_loss, global_step=global_step)
        adam_vars = [var for var in tf.all_variables() if 'Adam' in var.name]
        reset_optimizer_op = tf.variables_initializer(adam_vars)
        return train_op, reset_optimizer_op
//  [python]
def get_train_op(self, total_loss, global_step, learning_rate):
        """
        Return train_op
        """
        optimizer = tf.train.AdamOptimizer(learning_rate)
        train_op = optimizer.minimize(total_loss, global_step=global_step)
        adam_vars = [var for var in tf.global_variables() if 'Adam' in var.name]
        reset_optimizer_op = tf.variables_initializer(adam_vars)
        return train_op, reset_optimizer_op
//  [python]
def get_all_variables_in_graph(self):
        print('################### Variables of the graph')
        from collections import OrderedDict
        var_dict = OrderedDict()
        for var in tf.all_variables():
            if var.op.name not in var_dict.keys() and var.shape != () and "Adam" not in var.op.name:
                print("'" + str(var.op.name) + "': " + str(var.op.name).replace('/', '_') + "# " + str(var.shape))
                key = var.op.name
                # x = var.eval(self.sess)
                var_dict[key] = var.shape
        print('Finished Display all variables')
//  [python]
def get_all_variables_in_graph(self):
        print('################### Variables of the graph')
        from collections import OrderedDict
        var_dict = OrderedDict()
        for var in tf.global_variables():
            if var.op.name not in var_dict.keys() and var.shape != () and "Adam" not in var.op.name:
                print("'" + str(var.op.name) + "': " + str(var.op.name).replace('/', '_') + "# " + str(var.shape))
                key = var.op.name
                # x = var.eval(self.sess)
                var_dict[key] = var.shape
        print('Finished Display all variables')
//  [python]
def begin(self):
    # TODO(karishmamalkan): Add logging for when variables are loaded
    variable_references = {var.name: var for var in tf.all_variables()}
    variable_mappings = {}
    vars_in_chk = tf.train.list_variables(self.checkpoint_path)
    for (var, _) in vars_in_chk:
      variable_mappings[var] = variable_references[
          var.replace(self.chk_scopename, self.graph_scopename) + ":0"]
    tf.train.init_from_checkpoint(self.checkpoint_path, variable_mappings)
//  [python]
def begin(self):
    # TODO(karishmamalkan): Add logging for when variables are loaded
    variable_references = {var.name: var for var in tf.global_variables()}
    variable_mappings = {}
    vars_in_chk = tf.train.list_variables(self.checkpoint_path)
    for (var, _) in vars_in_chk:
      variable_mappings[var] = variable_references[
          var.replace(self.chk_scopename, self.graph_scopename) + ":0"]
    tf.train.init_from_checkpoint(self.checkpoint_path, variable_mappings)
//  [python]
def _init_uninitialized(sess):
  """Initializes all uninitialized variables and returns them as a list."""
  variables = tf.global_variables()
  if not variables: return []  # sess.run() barfs on empty list
  is_initialized = sess.run([tf.is_variable_initialized(v) for v in variables])
  needs_init = [v for v, i in zip(variables, is_initialized) if not i]
  if not needs_init: return []
  sess.run(tf.variables_initializer(needs_init))
  return needs_init
//  [python]
def initialize(self,ckpt_used):
        if os.path.exists(self.logging_directory) == False:
            os.makedirs(self.logging_directory)
        if os.path.exists(self.working_directory) == False:
            os.makedirs(self.working_directory)
        tf.gfile.DeleteRecursively(self.logging_directory)
        #used after create_model
        self.saver = tf.train.Saver(tf.all_variables(),max_to_keep=self.max_to_keep)
        self.writer = tf.summary.FileWriter(self.logging_directory, tf.get_default_graph(),
                                            max_queue=self.max_queue)
        self.init_op = tf.global_variables_initializer()

        if 'pretrained_model' in self.gConfig:
            self.saver.restore(self.session, self.gConfig['pretrained_model'])
        ckpt = tf.train.get_checkpoint_state(self.gConfig['working_directory'])
        if ckpt and ckpt.model_checkpoint_path and ckpt_used:
            print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
            self.saver.restore(self.session, ckpt.model_checkpoint_path)
            graph = tf.get_default_graph()

        else:
            print("Created model with fresh parameters.")
            self.session.run(self.init_op)
            graph = tf.get_default_graph()
            self.debug_info()
//  [python]
def initialize(self,ckpt_used):
        if os.path.exists(self.logging_directory) == False:
            os.makedirs(self.logging_directory)
        if os.path.exists(self.working_directory) == False:
            os.makedirs(self.working_directory)
        tf.gfile.DeleteRecursively(self.logging_directory)
        #used after create_model
        self.saver = tf.train.Saver(tf.global_variables(),max_to_keep=self.max_to_keep)
        self.writer = tf.summary.FileWriter(self.logging_directory, tf.get_default_graph(),
                                            max_queue=self.max_queue)
        self.init_op = tf.global_variables_initializer()

        if 'pretrained_model' in self.gConfig:
            self.saver.restore(self.session, self.gConfig['pretrained_model'])
        ckpt = tf.train.get_checkpoint_state(self.gConfig['working_directory'])
        if ckpt and ckpt.model_checkpoint_path and ckpt_used:
            print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
            self.saver.restore(self.session, ckpt.model_checkpoint_path)
            graph = tf.get_default_graph()

        else:
            print("Created model with fresh parameters.")
            self.session.run(self.init_op)
            graph = tf.get_default_graph()
            self.debug_info()
//  [python]
def get_params_internal(self, all_params=False, **tags):
        if tags.get('trainable', False):
            params = tf.trainable_variables()
        else:
            params = tf.global_variables()

        params = [p for p in params if p.name.startswith('mean_network') or p.name.startswith('output_std_param')]
        params = [p for p in params if 'Adam' not in p.name]

        return params
//  [python]
def init_uninitialized_vars(target_vars: List[tf.Variable] = None) -> None:
    """Initialize all tf.Variables that have not already been initialized.

    Equivalent to the following, but more efficient and does not bloat the tf graph:
    tf.variables_initializer(tf.report_uninitialized_variables()).run()
    """
    assert_tf_initialized()
    if target_vars is None:
        target_vars = tf.global_variables()

    test_vars = []
    test_ops = []

    with tf.control_dependencies(None):  # ignore surrounding control_dependencies
        for var in target_vars:
            assert is_tf_expression(var)

            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(":0", "/IsVariableInitialized:0"))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)

                with absolute_name_scope(var.name.split(":")[0]):
                    test_ops.append(tf.is_variable_initialized(var))

    init_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]
    run([var.initializer for var in init_vars])
//  [python]
def clean_ckpt(_):
    input_ckpt = FLAGS.clean_input_ckpt
    output_model_dir = FLAGS.clean_output_model_dir

    tf.reset_default_graph()

    var_list = tf.train.list_variables(input_ckpt)
    var_values, var_dtypes = {}, {}
    for (name, shape) in var_list:
        if not name.startswith("global_step") and "adam" not in name.lower():
            var_values[name] = None
            tf.logging.info("Include {}".format(name))
        else:
            tf.logging.info("Exclude {}".format(name))

    tf.logging.info("Loading from {}".format(input_ckpt))
    reader = tf.train.load_checkpoint(input_ckpt)
    for name in var_values:
        tensor = reader.get_tensor(name)
        var_dtypes[name] = tensor.dtype
        var_values[name] = tensor

    with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
        tf_vars = [
            tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
            for v in var_values
        ]
    placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
    assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
    global_step = tf.Variable(
        0, name="global_step", trainable=False, dtype=tf.int64)
    saver = tf.train.Saver(tf.all_variables())

    if not tf.gfile.Exists(output_model_dir):
        tf.gfile.MakeDirs(output_model_dir)

    # Build a model consisting only of variables, set them to the average values.
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                               six.iteritems(var_values)):
            sess.run(assign_op, {p: value})

        # Use the built saver to save the averaged checkpoint.
        saver.save(sess, join(output_model_dir, "model.ckpt"),
                   global_step=global_step)
//  [python]
def clean_ckpt(_):
    input_ckpt = FLAGS.clean_input_ckpt
    output_model_dir = FLAGS.clean_output_model_dir

    tf.reset_default_graph()

    var_list = tf.train.list_variables(input_ckpt)
    var_values, var_dtypes = {}, {}
    for (name, shape) in var_list:
        if not name.startswith("global_step") and "adam" not in name.lower():
            var_values[name] = None
            tf.logging.info("Include {}".format(name))
        else:
            tf.logging.info("Exclude {}".format(name))

    tf.logging.info("Loading from {}".format(input_ckpt))
    reader = tf.train.load_checkpoint(input_ckpt)
    for name in var_values:
        tensor = reader.get_tensor(name)
        var_dtypes[name] = tensor.dtype
        var_values[name] = tensor

    with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
        tf_vars = [
            tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
            for v in var_values
        ]
    placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
    assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
    global_step = tf.Variable(
        0, name="global_step", trainable=False, dtype=tf.int64)
    saver = tf.train.Saver(tf.global_variables())

    if not tf.gfile.Exists(output_model_dir):
        tf.gfile.MakeDirs(output_model_dir)

    # Build a model consisting only of variables, set them to the average values.
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                               six.iteritems(var_values)):
            sess.run(assign_op, {p: value})

        # Use the built saver to save the averaged checkpoint.
        saver.save(sess, join(output_model_dir, "model.ckpt"),
                   global_step=global_step)
//  [python]
def avg_checkpoints(model_dir, output_model_dir, last_k):
    tf.reset_default_graph()

    checkpoint_state = tf.train.get_checkpoint_state(model_dir)
    checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
    var_list = tf.train.list_variables(checkpoints[0])
    var_values, var_dtypes = {}, {}
    for (name, shape) in var_list:
        if not name.startswith("global_step"):
            var_values[name] = np.zeros(shape)
    for checkpoint in checkpoints:
        reader = tf.train.load_checkpoint(checkpoint)
        for name in var_values:
            tensor = reader.get_tensor(name)
            var_dtypes[name] = tensor.dtype
            var_values[name] += tensor
        tf.logging.info("Read from checkpoint %s", checkpoint)
    for name in var_values:  # Average.
        var_values[name] /= len(checkpoints)

    with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
        tf_vars = [
            tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
            for v in var_values
        ]
    placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
    assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
    global_step = tf.Variable(
        0, name="global_step", trainable=False, dtype=tf.int64)
    saver = tf.train.Saver(tf.all_variables())

    # Build a model consisting only of variables, set them to the average values.
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                               six.iteritems(var_values)):
            sess.run(assign_op, {p: value})
        # Use the built saver to save the averaged checkpoint.
        saver.save(sess, join(output_model_dir, "model.ckpt"),
                   global_step=global_step)
//  [python]
def avg_checkpoints(model_dir, output_model_dir, last_k):
    tf.reset_default_graph()

    checkpoint_state = tf.train.get_checkpoint_state(model_dir)
    checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
    var_list = tf.train.list_variables(checkpoints[0])
    var_values, var_dtypes = {}, {}
    for (name, shape) in var_list:
        if not name.startswith("global_step"):
            var_values[name] = np.zeros(shape)
    for checkpoint in checkpoints:
        reader = tf.train.load_checkpoint(checkpoint)
        for name in var_values:
            tensor = reader.get_tensor(name)
            var_dtypes[name] = tensor.dtype
            var_values[name] += tensor
        tf.logging.info("Read from checkpoint %s", checkpoint)
    for name in var_values:  # Average.
        var_values[name] /= len(checkpoints)

    with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
        tf_vars = [
            tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
            for v in var_values
        ]
    placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
    assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
    global_step = tf.Variable(
        0, name="global_step", trainable=False, dtype=tf.int64)
    saver = tf.train.Saver(tf.global_variables())

    # Build a model consisting only of variables, set them to the average values.
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                               six.iteritems(var_values)):
            sess.run(assign_op, {p: value})
        # Use the built saver to save the averaged checkpoint.
        saver.save(sess, join(output_model_dir, "model.ckpt"),
                   global_step=global_step)
//  [python]
def init_uninited_vars(vars=None):
    if vars is None: vars = tf.global_variables()
    test_vars = []; test_ops = []
    with tf.control_dependencies(None): # ignore surrounding control_dependencies
        for var in vars:
            assert is_tf_expression(var)
            try:
                tf.get_default_graph().get_tensor_by_name(var.name.replace(':0', '/IsVariableInitialized:0'))
            except KeyError:
                # Op does not exist => variable may be uninitialized.
                test_vars.append(var)
                with absolute_name_scope(var.name.split(':')[0]):
                    test_ops.append(tf.is_variable_initialized(var))
    init_vars = [var for var, inited in zip(test_vars, run(test_ops)) if not inited]
    run([var.initializer for var in init_vars])
//  [python]
def get_var_list(self):
        var_list = tf.compat.v1.global_variables()
        return var_list
//  [python]
def get_var_list(self):
        var_list = tf.global_variables()
        return var_list
//  [python]
def vars(self):
		return [var for var in tf.global_variables() if self.name in var.name]
//  [python]
def filter_variables(include=None, exclude=None):
  # Check arguments.
  if include is None:
    include = (r'.*',)
  if exclude is None:
    exclude = ()
  if not isinstance(include, (tuple, list)):
    include = (include,)
  if not isinstance(exclude, (tuple, list)):
    exclude = (exclude,)
  # Compile regexes.
  include = [re.compile(regex) for regex in include]
  exclude = [re.compile(regex) for regex in exclude]
  variables = tf.global_variables()
  if not variables:
    raise RuntimeError('There are no variables to filter.')
  # Check regexes.
  for regex in include:
    message = "Regex r'{}' does not match any variables in the graph.\n"
    message += 'All variables:\n'
    message += '\n'.join('- {}'.format(var.name) for var in variables)
    if not any(regex.match(variable.name) for variable in variables):
      raise RuntimeError(message.format(regex.pattern))
  # Filter variables.
  filtered = []
  for variable in variables:
    if not any(regex.match(variable.name) for regex in include):
      continue
    if any(regex.match(variable.name) for regex in exclude):
      continue
    filtered.append(variable)
  # Check result.
  if not filtered:
    message = 'No variables left after filtering.'
    message += '\nIncludes:\n' + '\n'.join(regex.pattern for regex in include)
    message += '\nExcludes:\n' + '\n'.join(regex.pattern for regex in exclude)
    raise RuntimeError(message)
  return filtered
//  [python]
def init_rest_var(sess):
    uninit_vars = []
    for var in tf.global_variables():
        try:
            sess.run(var)
        except tf.errors.FailedPreconditionError:
            uninit_vars.append(var)
    sess.run(tf.variables_initializer(uninit_vars))
//  [python]
def initialize_uninitialized(sess=None):
    """
    Initialize unitialized variables, doesn't affect those already initialized
    :param sess: in which session to initialize stuff. Defaults to tf.get_default_session()
    """
    sess = sess or tf.get_default_session()
    assert sess is not None, "please make sure you defined a default TF session"
    global_vars          = tf.global_variables()
    is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def sync_globals_zero_init_op(num_comms=2, prereduce=0):
    ops  = list()
    prev = []
    with tf.device("/gpu:0"):
        for var in tf.trainable_variables() + tf.global_variables():
            if var.dtype.base_dtype not in [tf.float32, tf.float16]:
                cast_back = True
                to_reduce = tf.cast(var, tf.float32)
            else:
                to_reduce = var
                cast_back = False
            with tf.control_dependencies(prev):
                reduced = allreduce(to_reduce, num_comms=num_comms, prereduce=prereduce)
            if cast_back:
                reduced = tf.cast(reduced, var.dtype.base_dtype)
            op = tf.assign(var, reduced)
            prev = [op]
            ops.append(op)

    return tf.group(*ops)
//  [python]
def sync_globals_zero_init_op(num_comms=2, prereduce=0):
    ops  = list()
    prev = []
    with tf.device("/gpu:0"):
        for var in tf.global_variables():
            if var.dtype.base_dtype not in [tf.float32, tf.float16]:
                cast_back = True
                to_reduce = tf.cast(var, tf.float32)
            else:
                to_reduce = var
                cast_back = False
            with tf.control_dependencies(prev):
                reduced = allreduce(to_reduce, num_comms=num_comms, prereduce=prereduce)
            if cast_back:
                reduced = tf.cast(reduced, var.dtype.base_dtype)
            op = tf.assign(var, reduced)
            prev = [op]
            ops.append(op)

    return tf.group(*ops)
//  [python]
def MNN_QAT_variables():
    all_vars = tf.compat.v1.global_variables()
    return [v for v in all_vars if 'MNN_QAT' in v.name]
//  [python]
def MNN_QAT_variables():
    all_vars = tf.global_variables()
    return [v for v in all_vars if 'MNN_QAT' in v.name]
//  [python]
preserve_org_names = ['Conv_6', 'Conv_14', 'Conv_22']


org_weights_mess = []
tf.Graph().as_default()
load = tf.train.import_meta_graph(org_weights_path + '.meta')
with tf.Session() as sess:
    load.restore(sess, org_weights_path)
    for var in tf.all_variables():
        var_name = var.op.name
        var_name_mess = str(var_name).split('/')
        var_shape = var.shape
        if flag.train_from_coco:
            if (var_name_mess[-1] not in ['weights', 'gamma', 'beta', 'moving_mean', 'moving_variance']) or \
                    (var_name_mess[1] == 'yolo-v3' and (var_name_mess[-2] in preserve_org_names)): continue
        org_weights_mess.append([var_name, var_shape])
        print("=> " + str(var_name).ljust(50), var_shape)
//  [python]
preserve_org_names = ['Conv_6', 'Conv_14', 'Conv_22']


org_weights_mess = []
tf.Graph().as_default()
load = tf.train.import_meta_graph(org_weights_path + '.meta')
with tf.Session() as sess:
    load.restore(sess, org_weights_path)
    for var in tf.global_variables():
        var_name = var.op.name
        var_name_mess = str(var_name).split('/')
        var_shape = var.shape
        if flag.train_from_coco:
            if (var_name_mess[-1] not in ['weights', 'gamma', 'beta', 'moving_mean', 'moving_variance']) or \
                    (var_name_mess[1] == 'yolo-v3' and (var_name_mess[-2] in preserve_org_names)): continue
        org_weights_mess.append([var_name, var_shape])
        print("=> " + str(var_name).ljust(50), var_shape)
//  [python]
def initialize_uninitialized(sess):
    global_vars          = tf.global_variables()
    is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
    print("initializing tensors:")
    print([str(i.name) for i in not_initialized_vars] )# only for testing
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def initialize_uninitialized(sess): # https://stackoverflow.com/questions/35164529/in-tensorflow-is-there-any-way-to-just-initialize-uninitialised-variables/35618160
    global_vars          = tf.global_variables()
    is_not_initialized   = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
    print("initializing tensors:")
    print([str(i.name) for i in not_initialized_vars] )# only for testing
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def init_uninitialized_variables(sess, list_of_variables=None):
  if list_of_variables is None:
    list_of_variables = tf.global_variables()
  uninitialized_variables = list(
      tf.get_variable(name) for name in sess.run(
          tf.report_uninitialized_variables(list_of_variables)))
  uninitialized_variables = tf.group(uninitialized_variables,
                                     tf.local_variables_initializer())
  sess.run(tf.variables_initializer(uninitialized_variables))
  return uninitialized_variables
//  [python]
def _init_optimizer(self):
        # 整理输出并计算loss
        logits = tf.transpose(self.decoder_logits_train, [1, 0, 2])
        targets = tf.transpose(self.decoder_train_targets, [1, 0])
        self.logits = tf.transpose(self.decoder_logits_train, [1, 0, 2])
        self.targets = tf.transpose(self.decoder_train_targets, [1, 0])

        self.loss = seq2seq.sequence_loss(logits=logits, targets=targets,
                                          weights=self.loss_weights)

        opt = tf.train.AdamOptimizer()
        self.train_op = opt.minimize(self.loss)

        # add
        params = tf.trainable_variables()
        self.gradient_norms = []
        self.updates = []

        gradients = tf.gradients(self.loss, params)
        clipped_gradients, norm = tf.clip_by_global_norm(gradients,
                                                         self.max_gradient_norm)
        self.gradient_norms.append(norm)
        self.updates.append(opt.apply_gradients(
            zip(clipped_gradients, params), global_step=self.global_step))

        self.saver = tf.train.Saver(tf.all_variables())
//  [python]
def _init_optimizer(self):
        # 整理输出并计算loss
        logits = tf.transpose(self.decoder_logits_train, [1, 0, 2])
        targets = tf.transpose(self.decoder_train_targets, [1, 0])
        self.logits = tf.transpose(self.decoder_logits_train, [1, 0, 2])
        self.targets = tf.transpose(self.decoder_train_targets, [1, 0])

        self.loss = seq2seq.sequence_loss(logits=logits, targets=targets,
                                          weights=self.loss_weights)

        opt = tf.train.AdamOptimizer()
        self.train_op = opt.minimize(self.loss)

        # add
        params = tf.trainable_variables()
        self.gradient_norms = []
        self.updates = []

        gradients = tf.gradients(self.loss, params)
        clipped_gradients, norm = tf.clip_by_global_norm(gradients,
                                                         self.max_gradient_norm)
        self.gradient_norms.append(norm)
        self.updates.append(opt.apply_gradients(
            zip(clipped_gradients, params), global_step=self.global_step))

        self.saver = tf.train.Saver(tf.global_variables())
//  [python]
def train(self):
        # Epoch 0: 0-epoch_size; saved as snapshot_1.ckpt
        # Epoch 1: epoch_size-2x; saved as snapshot_2.ckpt
        # ...
        # Epoch 9: saved as snapshot_10.ckpt
        # dec lr; Epoch 10: saved as snapshot_11.ckpt
        # ...
        # if you wanna test epoch before decrease lr, just test snapshot_10.ckpt

        self.logger.info('Initialize saver ...')
        # saver
        train_saver = Saver(self.sess, tf.global_variables(), self.cfg.model_dump_dir)

        self.logger.info('Initialize all variables ...')
        self.sess.run(tf.variables_initializer(tf.global_variables(), name='init'))

        # initialize weights
        self.load_weights('last_epoch' if self.cfg.continue_train else self.cfg.init_model)

        self.logger.info('Start training ...')
        start_itr = self.cur_epoch * self.cfg.epoch_size + 1
        nr_itrs = self.cfg.nr_gpus*self.cfg.batch_size
        for itr in range(start_itr, self.cfg.max_itr + nr_itrs, nr_itrs):
            self.global_timer.tic()

            itrs = np.arange(itr, itr+nr_itrs)
            self.cur_epoch = itrs[-1] // self.cfg.epoch_size

            setproctitle.setproctitle('train ' + self.cfg.proj_name + ' epoch:' + str(self.cur_epoch))

            # apply current learning policy
            cur_lr = self.cfg.get_lr(itrs[-1])
            if not approx_equal(cur_lr, self.lr_eval):
                print(self.lr_eval, cur_lr)
                self.sess.run(tf.assign(self.lr, cur_lr))

            # input data
            self.read_timer.tic()
            feed_dict = self.next_feed()
            self.read_timer.toc()

            # train one step
            self.timer.tic()
            _, self.lr_eval, *summary_res = self.sess.run(
                [self.graph_ops[0], self.lr, *self.summary_dict.values()], feed_dict=feed_dict)
            self.timer.toc()

            iter_summary = dict()
            for i, k in enumerate(self.summary_dict.keys()):
                iter_summary[k] = summary_res[i]

            screen = [
                'Epoch %d itr %d/%d:' % (self.cur_epoch, itrs[-1], self.cfg.epoch_size),
                'lr: %g' % (self.lr_eval),
                'speed: %.2f(%.2fs r%.2f)s/itr' % (
                    self.global_timer.average_time, self.timer.average_time, self.read_timer.average_time),
                '%.2fh/epoch' % (self.global_timer.average_time / 3600. * self.cfg.epoch_size / nr_itrs),
                ' '.join(map(lambda x: '%s: %.4f' % (x[0], x[1]), iter_summary.items())),
            ]

            #TODO(display stall?)
            if np.any(itrs % (self.cfg.display) == 0):
                self.logger.info(' '.join(screen))

            if np.any(itrs % self.cfg.epoch_size == 0):
                train_saver.save_model(self.cur_epoch)

            self.global_timer.toc()
//  [python]
def _get_variable_by_name(self, name):
        return [v for v in tf.compat.v1.global_variables() if v.op.name == name][0]
//  [python]
def _get_variable_by_name(self, name):
        return [v for v in tf.global_variables() if v.op.name == name][0]
//  [python]
def initialize_uninitialized_vars(sess):
  import itertools
  from itertools import compress
  global_vars = tf.global_variables()
  is_not_initialized = sess.run([~(tf.is_variable_initialized(var)) \
                                  for var in global_vars])
  not_initialized_vars = list(compress(global_vars, is_not_initialized))

  if len(not_initialized_vars):
    sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def get_global_variable_by_name(name):
	"""Returns the global variable of given name.

	name : the name of the global variable
	"""
	# return [v for v in tf.variables() if v.name == name][0]
	return [v for v in tf.global_variables() if v.name == name][0]
//  [python]
def main(args):
  
    network = importlib.import_module(args.model_def, 'inference')

    # Start with a gray image with a little noise
    np.random.seed(seed=args.seed)
    img_noise = np.random.uniform(size=(args.image_size,args.image_size,3)) + 100.0
  
    sess = tf.Session()
  
    t_input = tf.placeholder(np.float32, shape=(args.image_size,args.image_size,3), name='input') # define the input tensor
    image_mean = 117.0
    t_preprocessed = tf.expand_dims(t_input-image_mean, 0)
     
    # Build the inference graph
    network.inference(t_preprocessed, 1.0, 
            phase_train=True, weight_decay=0.0)
      
    # Create a saver for restoring variables
    saver = tf.train.Saver(tf.all_variables())
  
    # Restore the parameters
    saver.restore(sess, args.model_file)
  
    layers = [op.name for op in tf.get_default_graph().get_operations() if op.type=='Conv2D']
    feature_nums = {layer: int(T(layer).get_shape()[-1]) for layer in layers}
  
    print('Number of layers: %d' % len(layers))
  
    for layer in sorted(feature_nums.keys()):
        print('%s%d' % ((layer+': ').ljust(40), feature_nums[layer]))
  
    # Picking some internal layer. Note that we use outputs before applying the ReLU nonlinearity
    # to have non-zero gradients for features with negative initial activations.
    layer = 'InceptionResnetV1/Repeat_2/block8_3/Conv2d_1x1/Conv2D'
    #layer = 'incept4b/in4_conv1x1_31/Conv2D'
    result_dir = '../data/'
    print('Number of features in layer "%s": %d' % (layer, feature_nums[layer]))
    channels = range(feature_nums[layer])
    np.random.shuffle(channels)
    for i in range(32):
        print('Rendering feature %d' % channels[i])
        channel = channels[i]
        img = render_naive(sess, t_input, T(layer)[:,:,:,channel], img_noise)
        filename = '%s_%03d.png' % (layer.replace('/', '_'), channel)
        misc.imsave(os.path.join(result_dir, filename), img)
//  [python]
def main(args):
  
    network = importlib.import_module(args.model_def, 'inference')

    # Start with a gray image with a little noise
    np.random.seed(seed=args.seed)
    img_noise = np.random.uniform(size=(args.image_size,args.image_size,3)) + 100.0
  
    sess = tf.Session()
  
    t_input = tf.placeholder(np.float32, shape=(args.image_size,args.image_size,3), name='input') # define the input tensor
    image_mean = 117.0
    t_preprocessed = tf.expand_dims(t_input-image_mean, 0)
     
    # Build the inference graph
    network.inference(t_preprocessed, 1.0, 
            phase_train=True, weight_decay=0.0)
      
    # Create a saver for restoring variables
    saver = tf.train.Saver(tf.global_variables())
  
    # Restore the parameters
    saver.restore(sess, args.model_file)
  
    layers = [op.name for op in tf.get_default_graph().get_operations() if op.type=='Conv2D']
    feature_nums = {layer: int(T(layer).get_shape()[-1]) for layer in layers}
  
    print('Number of layers: %d' % len(layers))
  
    for layer in sorted(feature_nums.keys()):
        print('%s%d' % ((layer+': ').ljust(40), feature_nums[layer]))
  
    # Picking some internal layer. Note that we use outputs before applying the ReLU nonlinearity
    # to have non-zero gradients for features with negative initial activations.
    layer = 'InceptionResnetV1/Repeat_2/block8_3/Conv2d_1x1/Conv2D'
    #layer = 'incept4b/in4_conv1x1_31/Conv2D'
    result_dir = '../data/'
    print('Number of features in layer "%s": %d' % (layer, feature_nums[layer]))
    channels = range(feature_nums[layer])
    np.random.shuffle(channels)
    for i in range(32):
        print('Rendering feature %d' % channels[i])
        channel = channels[i]
        img = render_naive(sess, t_input, T(layer)[:,:,:,channel], img_noise)
        filename = '%s_%03d.png' % (layer.replace('/', '_'), channel)
        misc.imsave(os.path.join(result_dir, filename), img)
//  [python]
def initialize_uninitialized_global_variables(sess):
    """
    Only initializes the variables of a TensorFlow session that were not
    already initialized.
    :param sess: the TensorFlow session
    :return:
    """
    # List all global variables
    global_vars = tf.global_variables()

    # Find initialized status for all variables
    is_var_init = [tf.is_variable_initialized(var) for var in global_vars]
    is_initialized = sess.run(is_var_init)

    # List all variables that were not initialized previously
    not_initialized_vars = [var for (var, init) in
                            zip(global_vars, is_initialized) if not init]

    # Initialize all uninitialized variables found, if any
    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def get_vars(scope):
    return [x for x in tf.compat.v1.global_variables() if scope in x.name]
//  [python]
def get_vars(scope):
    return [x for x in tf.global_variables() if scope in x.name]
//  [python]
def generate_model_info():
    op_stats = OpStats()
    tensor_stats = TensorStats()
    all_ops = tf.get_default_graph().get_operations()
    op_stats.op_count = len(all_ops)
    for op in all_ops:
        if "update_" in op.name:  # Ops with update_ executed on PS
            op_stats.update_op_count += 1
        if op.name.endswith("/read") or op.name.endswith(
            "/Read/ReadVariableOp"
        ):
            op_stats.read_op_count += 1
    variables = tf.all_variables()
    tensor_stats.variable_count = len(variables)
    for var in variables:
        shape = var.get_shape().as_list()
        if hasattr(var, "key_dtype"):  # The unique attr for KV embedding
            tensor_stats.kv_embedding_dims.append(int(shape[-1]))
        else:
            var_size = 1
            for dimesion in shape:
                var_size *= dimesion
            tensor_stats.total_variable_size += var_size
            tensor_stats.max_variable_size = max(
                tensor_stats.max_variable_size, var_size
            )
    model_info = ModelInfo(tensor_stats, op_stats)
    return model_info
//  [python]
def generate_model_info():
    op_stats = OpStats()
    tensor_stats = TensorStats()
    all_ops = tf.get_default_graph().get_operations()
    op_stats.op_count = len(all_ops)
    for op in all_ops:
        if "update_" in op.name:  # Ops with update_ executed on PS
            op_stats.update_op_count += 1
        if op.name.endswith("/read") or op.name.endswith(
            "/Read/ReadVariableOp"
        ):
            op_stats.read_op_count += 1
    variables = tf.global_variables()
    tensor_stats.variable_count = len(variables)
    for var in variables:
        shape = var.get_shape().as_list()
        if hasattr(var, "key_dtype"):  # The unique attr for KV embedding
            tensor_stats.kv_embedding_dims.append(int(shape[-1]))
        else:
            var_size = 1
            for dimesion in shape:
                var_size *= dimesion
            tensor_stats.total_variable_size += var_size
            tensor_stats.max_variable_size = max(
                tensor_stats.max_variable_size, var_size
            )
    model_info = ModelInfo(tensor_stats, op_stats)
    return model_info
//  [python]
def __init__(self,sess,param):
        self.step = 0
        self.__session = sess
        self.is_training=True
        self.__learn_rate = param["learn_rate"]
        self.__learn_rate=param["learn_rate"]
        self.__max_to_keep=param["max_to_keep"]
        self.__checkPoint_dir = param["checkPoint_dir"]
        self.__restore = param["b_restore"]
        self.__mode= param["mode"]
        self.is_training=True
        self.__batch_size = param["batch_size"]
        if  self.__mode is "savaPb" :
            self.__batch_size = 1
        ################ Building graph
        with self.__session.as_default():
            self.build_model()
        ###############参数初始化，或者读入参数
        with self.__session.as_default():
            self.init_op.run()
            self.__saver = tf.train.Saver(tf.all_variables(), max_to_keep=self.__max_to_keep)
            # Loading last save if needed
            if self.__restore:
                ckpt = tf.train.latest_checkpoint(self.__checkPoint_dir)
                if ckpt:
                    self.step = int(ckpt.split('-')[1])
                    self.__saver.restore(self.__session, ckpt)
                    print('Restoring from epoch:{}'.format( self.step))
                    self.step+=1
//  [python]
def __init__(self,sess,param):
        self.step = 0
        self.__session = sess
        self.is_training=True
        self.__learn_rate = param["learn_rate"]
        self.__learn_rate=param["learn_rate"]
        self.__max_to_keep=param["max_to_keep"]
        self.__checkPoint_dir = param["checkPoint_dir"]
        self.__restore = param["b_restore"]
        self.__mode= param["mode"]
        self.is_training=True
        self.__batch_size = param["batch_size"]
        if  self.__mode is "savaPb" :
            self.__batch_size = 1
        ################ Building graph
        with self.__session.as_default():
            self.build_model()
        ###############参数初始化，或者读入参数
        with self.__session.as_default():
            self.init_op.run()
            self.__saver = tf.train.Saver(tf.global_variables(), max_to_keep=self.__max_to_keep)
            # Loading last save if needed
            if self.__restore:
                ckpt = tf.train.latest_checkpoint(self.__checkPoint_dir)
                if ckpt:
                    self.step = int(ckpt.split('-')[1])
                    self.__saver.restore(self.__session, ckpt)
                    print('Restoring from epoch:{}'.format( self.step))
                    self.step+=1
//  [python]
def testNoBatchNormScaleByDefault(self):
    height, width = 299, 299
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with tf.contrib.slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
      inception.inception_resnet_v2(inputs, num_classes, is_training=False)

    self.assertEqual(tf.global_variables('.*/BatchNorm/gamma:0$'), [])
//  [python]
def optimistic_restore(session, save_file):
    reader = tf.train.NewCheckpointReader(save_file)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted([(var.name, var.name.split(':')[0]) for var in tf.all_variables()
            if var.name.split(':')[0] in saved_shapes])
    restore_vars = []
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = tf.get_variable(saved_var_name)
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
    saver = tf.train.Saver(restore_vars)
    saver.restore(session, save_file)
//  [python]
def optimistic_restore(session, save_file):
    reader = tf.train.NewCheckpointReader(save_file)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted([(var.name, var.name.split(':')[0]) for var in tf.global_variables()
            if var.name.split(':')[0] in saved_shapes])
    restore_vars = []
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = tf.get_variable(saved_var_name)
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
    saver = tf.train.Saver(restore_vars)
    saver.restore(session, save_file)
//  [python]
def testBatchNormScale(self):
    height, width = 299, 299
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with tf.contrib.slim.arg_scope(
        inception.inception_resnet_v2_arg_scope(batch_norm_scale=True)):
      inception.inception_resnet_v2(inputs, num_classes, is_training=False)

    gamma_names = set(
        v.op.name for v in tf.compat.v1.global_variables('.*/BatchNorm/gamma:0$'))
    self.assertGreater(len(gamma_names), 0)
    for v in tf.compat.v1.global_variables('.*/BatchNorm/moving_mean:0$'):
      self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names)
//  [python]
def testBatchNormScale(self):
    height, width = 299, 299
    num_classes = 1000
    inputs = tf.placeholder(tf.float32, (1, height, width, 3))
    with tf.contrib.slim.arg_scope(
        inception.inception_resnet_v2_arg_scope(batch_norm_scale=True)):
      inception.inception_resnet_v2(inputs, num_classes, is_training=False)

    gamma_names = set(
        v.op.name for v in tf.global_variables('.*/BatchNorm/gamma:0$'))
    self.assertGreater(len(gamma_names), 0)
    for v in tf.global_variables('.*/BatchNorm/moving_mean:0$'):
      self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names)
//  [python]
def restore_from_classification_checkpoint_fn(
      self,
      first_stage_feature_extractor_scope,
      second_stage_feature_extractor_scope):
    """Returns a map of variables to load from a foreign checkpoint.

    Note that this overrides the default implementation in
    faster_rcnn_meta_arch.FasterRCNNFeatureExtractor which does not work for
    PNASNet checkpoints.

    Args:
      first_stage_feature_extractor_scope: A scope name for the first stage
        feature extractor.
      second_stage_feature_extractor_scope: A scope name for the second stage
        feature extractor.

    Returns:
      A dict mapping variable names (to load from a checkpoint) to variables in
      the model graph.
    """
    variables_to_restore = {}
    for variable in tf.global_variables():
      if variable.op.name.startswith(
          first_stage_feature_extractor_scope):
        var_name = variable.op.name.replace(
            first_stage_feature_extractor_scope + '/', '')
        var_name += '/ExponentialMovingAverage'
        variables_to_restore[var_name] = variable
      if variable.op.name.startswith(
          second_stage_feature_extractor_scope):
        var_name = variable.op.name.replace(
            second_stage_feature_extractor_scope + '/', '')
        var_name += '/ExponentialMovingAverage'
        variables_to_restore[var_name] = variable
    return variables_to_restore
//  [python]
def print_variable_summary():
    import pprint
    variables = sorted([[v.name, v.get_shape()] for v in tf.global_variables()])
    pprint.pprint(variables)
//  [python]
def add_saver(self):
    """Adds a Saver for all variables in the graph."""
    logging.info('Saving variables:\n\t%s',
                 '\n\t'.join([x.name for x in tf.global_variables()]))
    self.saver = tf.train.Saver(
        var_list=[x for x in tf.global_variables()],
        write_version=saver_pb2.SaverDef.V1)
//  [python]
def initialize_uninitialized(sess=None):
    """
    Initialize unitialized variables, doesn't affect those already initialized
    :param sess: in which session to initialize stuff. Defaults to tf.get_default_session()
    """
    sess = sess or tf.get_default_session()
    global_vars = tf.all_variables()
    is_not_initialized = sess.run(
        [tf.is_variable_initialized(var) for var in global_vars]
    )
    not_initialized_vars = [
        v for (v, f)
        in zip(global_vars, is_not_initialized)
        if not f
    ]

    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def initialize_uninitialized(sess=None):
    """
    Initialize unitialized variables, doesn't affect those already initialized
    :param sess: in which session to initialize stuff. Defaults to tf.get_default_session()
    """
    sess = sess or tf.get_default_session()
    global_vars = tf.global_variables()
    is_not_initialized = sess.run(
        [tf.is_variable_initialized(var) for var in global_vars]
    )
    not_initialized_vars = [
        v for (v, f)
        in zip(global_vars, is_not_initialized)
        if not f
    ]

    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))