jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def get_facenet_variables_to_restore():
    facenet_variables = []
    for var in tf.global_variables():
        if var.name.startswith('Inception'):
            if 'Adam' not in var.name:
                facenet_variables.append(var)
    return facenet_variables
//  [python]
def _get_restore_vars(self, ckpt_var_map_path):
    """Restore by specify variable map between graph variables and ckpt variables.

    Args:
      ckpt_var_map_path: variable map from graph variables to variables in a checkpoint
          each line consists of: variable name in graph  variable name in ckpt

    Returns:
      the list of variables which need to restore from checkpoint
    """
    # here must use global_variables, because variables such as moving_mean
    #  and moving_variance is usually not trainable in detection models
    all_vars = tf.global_variables()
    PARTITION_PATTERN = '/part_[0-9]+'
    VAR_SUFIX_PATTERN = ':[0-9]$'

    name2var = {}
    for one_var in all_vars:
      var_name = re.sub(VAR_SUFIX_PATTERN, '', one_var.name)
      if re.search(PARTITION_PATTERN,
                   var_name) and one_var._save_slice_info is not None:
        var_name = re.sub(PARTITION_PATTERN, '', var_name)
        is_part = True
      else:
        is_part = False
      if var_name in name2var:
        assert is_part, 'multiple vars: %s' % var_name
        name2var[var_name].append(one_var)
      else:
        name2var[var_name] = [one_var] if is_part else one_var

    if ckpt_var_map_path != '':
      if not gfile.Exists(ckpt_var_map_path):
        logging.warning('%s not exist' % ckpt_var_map_path)
        return name2var

      # load var map
      name_map = {}
      with gfile.GFile(ckpt_var_map_path, 'r') as fin:
        for one_line in fin:
          one_line = one_line.strip()
          line_tok = [x for x in one_line.split() if x != '']
          if len(line_tok) != 2:
            logging.warning('Failed to process: %s' % one_line)
            continue
          name_map[line_tok[0]] = line_tok[1]
      update_map = {}
      old_keys = []
      for var_name in name2var:
        if var_name in name_map:
          in_ckpt_name = name_map[var_name]
          update_map[in_ckpt_name] = name2var[var_name]
          old_keys.append(var_name)
      for tmp_key in old_keys:
        del name2var[tmp_key]
      name2var.update(update_map)
      return name2var
    else:
      var_filter, scope_update = self.get_restore_filter()
      if var_filter is not None:
        name2var = {
            var_name: name2var[var_name]
            for var in name2var
            if var_filter.keep(var.name)
        }
      # drop scope prefix if necessary
      if scope_update is not None:
        name2var = {
            scope_update(var_name): name2var[var_name] for var_name in name2var
        }
      return name2var
//  [python]
def _restore_list(self):
        return [n for n in tf.compat.v1.global_variables()
                 if n.name != 'entities/_entity_embeddings:0']
//  [python]
def _restore_list(self):
        return [n for n in tf.global_variables()
                 if n.name != 'entities/_entity_embeddings:0']
//  [python]
def save_network(self, tf_session, model_dir, checkpoint_name):
        saver = tf.train.Saver(max_to_keep=100000)
        vars = 0
        for v in tf.all_variables():
            vars += np.prod(v.get_shape().as_list())

        save_path = saver.save(tf_session, os.path.join(model_dir, "{0}.ckpt".format(checkpoint_name)))
        logging.info("Model saved to: {0}".format(save_path))
//  [python]
def save_network(self, tf_session, model_dir, checkpoint_name):
        saver = tf.train.Saver(max_to_keep=100000)
        vars = 0
        for v in tf.global_variables():
            vars += np.prod(v.get_shape().as_list())

        save_path = saver.save(tf_session, os.path.join(model_dir, "{0}.ckpt".format(checkpoint_name)))
        logging.info("Model saved to: {0}".format(save_path))
//  [python]
def find_variable_handler_by_name(self, var_name):
        for n in tf.global_variables():
            if n.name == var_name:
                return n
//  [python]
def __init__(self, net, data):
        self.net = net
        self.data = data
        self.weights_file = cfg.WEIGHTS_FILE
        self.max_iter = cfg.MAX_ITER
        self.initial_learning_rate = cfg.LEARNING_RATE
        self.decay_steps = cfg.DECAY_STEPS
        self.decay_rate = cfg.DECAY_RATE
        self.staircase = cfg.STAIRCASE
        self.summary_iter = cfg.SUMMARY_ITER
        self.save_iter = cfg.SAVE_ITER
        self.output_dir = os.path.join(
            cfg.OUTPUT_DIR, datetime.datetime.now().strftime('%Y_%m_%d_%H_%M'))
        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)
        self.save_cfg()

        self.variable_to_restore = tf.global_variables()
        self.saver = tf.train.Saver(self.variable_to_restore, max_to_keep=None)
        self.ckpt_file = os.path.join(self.output_dir, 'yolo')
        self.summary_op = tf.summary.merge_all()
        self.writer = tf.summary.FileWriter(self.output_dir, flush_secs=60)

        self.global_step = tf.train.create_global_step()
        self.learning_rate = tf.train.exponential_decay(
            self.initial_learning_rate, self.global_step, self.decay_steps,
            self.decay_rate, self.staircase, name='learning_rate')
        self.optimizer = tf.train.GradientDescentOptimizer(
            learning_rate=self.learning_rate)
        self.train_op = slim.learning.create_train_op(
            self.net.total_loss, self.optimizer, global_step=self.global_step)

        gpu_options = tf.GPUOptions()
        config = tf.ConfigProto(gpu_options=gpu_options)
        self.sess = tf.Session(config=config)
        self.sess.run(tf.global_variables_initializer())

        if self.weights_file is not None:
            print('Restoring weights from: ' + self.weights_file)
            self.saver.restore(self.sess, self.weights_file)

        self.writer.add_graph(self.sess.graph)
//  [python]
def checkpoint_variables(self):
        """omit word embeddings and entity embeddings from being stored in checkpoint when they are fixed
        in order to save disk space. word emb are always fixed, entity emb are fixed when
        args.train_ent_vecs == False"""
        omit_variables = ['words/_word_embeddings:0']
        if not self.args.train_ent_vecs:
            omit_variables.append('entities/_entity_embeddings:0')
        variables = [n for n in tf.global_variables() if n.name not in omit_variables]
        print("checkpoint variables to restore:", variables)
        return variables
//  [python]
def savable_variables(self):
    """Returns a list/dict of savable variables to pass to tf.train.Saver."""
    return tf.global_variables()
//  [python]
def savable_variables(self):
    """Return the set of variables used for saving/loading the model."""
    params = []
    for v in tf.compat.v1.global_variables():
      split_name = v.name.split('/')
      if split_name[0] == 'v0' or not v.name.startswith('v'):
        params.append(v)
    return params
//  [python]
def savable_variables(self):
    """Return the set of variables used for saving/loading the model."""
    params = []
    for v in tf.global_variables():
      split_name = v.name.split('/')
      if split_name[0] == 'v0' or not v.name.startswith('v'):
        params.append(v)
    return params
//  [python]
def get_post_init_ops(self):
    # Copy initialized values for variables on GPU 0 to other GPUs.
    global_vars = tf.global_variables()
    var_by_name = dict([(v.name, v) for v in global_vars])
    post_init_ops = []
    copy_froms = set()
    skipped_vars = []
    for v in global_vars:
      split_name = v.name.split('/')
      # TODO(b/62630508): use more specific prefix than v or v0.
      if split_name[0] == 'v0' or not v.name.startswith('v'):
        skipped_vars.append(v)
        continue
      # Only vars starts with "v[number]" are synced.
      split_name[0] = 'v0'
      copy_from = var_by_name['/'.join(split_name)]
      copy_froms.add(copy_from)
      post_init_ops.append(v.assign(copy_from.read_value()))
    post_init_ops += self._warmup_ops
    # If copy-froms is empty, then all vars are actually saved.
    misc_utils.print_out('All copy-from vars(%d): ' % len(copy_froms))
    for gv in copy_froms:
      misc_utils.print_out(gv.name)
    misc_utils.print_out('All skippped vars(%d): ' % len(skipped_vars))
    for gv in skipped_vars:
      misc_utils.print_out(gv.name)
    assert len(skipped_vars) >= len(copy_froms)

    return post_init_ops
//  [python]
def restore_from_classification_checkpoint_fn(
      self,
      checkpoint_path,
      first_stage_feature_extractor_scope,
      second_stage_feature_extractor_scope):
    """Returns callable for loading a checkpoint into the tensorflow graph.

    Args:
      checkpoint_path: path to checkpoint to restore.
      first_stage_feature_extractor_scope: A scope name for the first stage
        feature extractor.
      second_stage_feature_extractor_scope: A scope name for the second stage
        feature extractor.

    Returns:
      a callable which takes a tf.Session as input and loads a checkpoint when
        run.
    """
    variables_to_restore = {}
    for variable in tf.global_variables():
      for scope_name in [first_stage_feature_extractor_scope,
                         second_stage_feature_extractor_scope]:
        if variable.op.name.startswith(scope_name):
          var_name = variable.op.name.replace(scope_name + '/', '')
          variables_to_restore[var_name] = variable
    variables_to_restore = (
        variables_helper.get_variables_available_in_checkpoint(
            variables_to_restore, checkpoint_path))
    saver = tf.train.Saver(variables_to_restore)
    def restore(sess):
      saver.restore(sess, checkpoint_path)
    return restore
//  [python]
def get_uninitialized_variables(variables=None, name=None):
    """
    Get uninitialized variables as a list.

    Args:
        variables (list[tf.Variable]): Collect only uninitialized variables
            within this list. If not specified, will collect all uninitialized
            variables within ``tf.GraphKeys.GLOBAL_VARIABLES`` collection.
        name (str): TensorFlow name scope of the graph nodes.

    Returns:
        list[tf.Variable]: Uninitialized variables.
    """
    sess = get_default_session_or_error()
    if variables is None:
        variables = tf.compat.v1.global_variables()
    else:
        variables = list(variables)
    with tf.name_scope(name, default_name='get_uninitialized_variables'):
        init_flag = sess.run(tf.stack(
            [tf.is_variable_initialized(v) for v in variables]
        ))
    return [v for v, f in zip(variables, init_flag) if not f]
//  [python]
def get_uninitialized_variables(variables=None, name=None):
    """
    Get uninitialized variables as a list.

    Args:
        variables (list[tf.Variable]): Collect only uninitialized variables
            within this list. If not specified, will collect all uninitialized
            variables within ``tf.GraphKeys.GLOBAL_VARIABLES`` collection.
        name (str): TensorFlow name scope of the graph nodes.

    Returns:
        list[tf.Variable]: Uninitialized variables.
    """
    sess = get_default_session_or_error()
    if variables is None:
        variables = tf.global_variables()
    else:
        variables = list(variables)
    with tf.name_scope(name, default_name='get_uninitialized_variables'):
        init_flag = sess.run(tf.stack(
            [tf.is_variable_initialized(v) for v in variables]
        ))
    return [v for v, f in zip(variables, init_flag) if not f]
//  [python]
def restore_fn(self, checkpoint_path, from_detection_checkpoint=True):
    """Returns callable for loading a checkpoint into the tensorflow graph.

    Args:
      checkpoint_path: path to checkpoint to restore.
      from_detection_checkpoint: whether to restore from a detection checkpoint
        (with compatible variable names) or to restore from a classification
        checkpoint for initialization prior to training.  Note that when
        from_detection_checkpoint=True, the current implementation only
        supports restoration from an (exactly) identical model (with exception
        of the num_classes parameter).

    Returns:
      a callable which takes a tf.Session as input and loads a checkpoint when
        run.
    """
    if not from_detection_checkpoint:
      return self._feature_extractor.restore_from_classification_checkpoint_fn(
          checkpoint_path,
          self.first_stage_feature_extractor_scope,
          self.second_stage_feature_extractor_scope)

    variables_to_restore = tf.all_variables()
    variables_to_restore.append(slim.get_or_create_global_step())
    # Only load feature extractor variables to be consistent with loading from
    # a classification checkpoint.
    first_stage_variables = tf.contrib.framework.filter_variables(
        variables_to_restore,
        include_patterns=[self.first_stage_feature_extractor_scope,
                          self.second_stage_feature_extractor_scope])

    saver = tf.train.Saver(first_stage_variables)

    def restore(sess):
      saver.restore(sess, checkpoint_path)
    return restore
//  [python]
def restore_fn(self, checkpoint_path, from_detection_checkpoint=True):
    """Returns callable for loading a checkpoint into the tensorflow graph.

    Args:
      checkpoint_path: path to checkpoint to restore.
      from_detection_checkpoint: whether to restore from a detection checkpoint
        (with compatible variable names) or to restore from a classification
        checkpoint for initialization prior to training.  Note that when
        from_detection_checkpoint=True, the current implementation only
        supports restoration from an (exactly) identical model (with exception
        of the num_classes parameter).

    Returns:
      a callable which takes a tf.Session as input and loads a checkpoint when
        run.
    """
    if not from_detection_checkpoint:
      return self._feature_extractor.restore_from_classification_checkpoint_fn(
          checkpoint_path,
          self.first_stage_feature_extractor_scope,
          self.second_stage_feature_extractor_scope)

    variables_to_restore = tf.global_variables()
    variables_to_restore.append(slim.get_or_create_global_step())
    # Only load feature extractor variables to be consistent with loading from
    # a classification checkpoint.
    first_stage_variables = tf.contrib.framework.filter_variables(
        variables_to_restore,
        include_patterns=[self.first_stage_feature_extractor_scope,
                          self.second_stage_feature_extractor_scope])

    saver = tf.train.Saver(first_stage_variables)

    def restore(sess):
      saver.restore(sess, checkpoint_path)
    return restore
//  [python]
def initialize_uninitialized(sess=None):
    """
    Initialize unitialized variables, doesn't affect those already initialized
    :param sess: in which session to initialize stuff. Defaults to tf.get_default_session()
    """
    sess = sess or tf.get_default_session()
    assert sess is not None, "please make sure you defined a default TF session"
    global_vars = tf.all_variables()
    is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def initialize_uninitialized(sess=None):
    """
    Initialize unitialized variables, doesn't affect those already initialized
    :param sess: in which session to initialize stuff. Defaults to tf.get_default_session()
    """
    sess = sess or tf.get_default_session()
    assert sess is not None, "please make sure you defined a default TF session"
    global_vars = tf.global_variables()
    is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
    not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]

    if len(not_initialized_vars):
        sess.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def swa():
    path_base = fsdb.models_dir()
    model_names = [
        "000393-lincoln",
        "000390-indus",
        "000404-hannibal",
        "000447-hawke",
        "000426-grief",
        "000431-lion",
        "000428-invincible",
        "000303-olympus",
        "000291-superb",
        "000454-victorious",
    ]
    model_names = model_names[:FLAGS.count]

    model_paths = [os.path.join(path_base, m) for m in model_names]

    # construct the graph
    features, labels = dual_net.get_inference_input()
    dual_net.model_fn(features, labels, tf.estimator.ModeKeys.PREDICT, FLAGS.flag_values_dict())

    # restore all saved weights
    meta_graph_def = meta_graph.read_meta_graph_file(model_paths[0] + '.meta')
    stored_var_names = set(
        [n.name for n in meta_graph_def.graph_def.node if n.op == 'VariableV2'])

    var_list = [v for v in tf.all_variables()
                if v.op.name in stored_var_names]
    var_list.sort(key=lambda v: v.op.name)

    print(stored_var_names)
    print(len(stored_var_names), len(var_list))

    sessions = [tf.Session() for _ in model_paths]
    saver = tf.train.Saver()
    for sess, model_path in zip(sessions, model_paths):
        saver.restore(sess, model_path)

    # Load all VariableV2s for each model.
    values = [sess.run(var_list) for sess in sessions]

    # Iterate over all variables average values from all models.
    all_assign = []
    for var, vals in zip(var_list, zip(*values)):
        print("{}x {}".format(len(vals), var))
        if var.name == "global_step:0":
            avg = vals[0]
            for val in vals:
                avg = tf.maximum(avg, val)
        else:
            avg = tf.add_n(vals) / len(vals)
            continue

        all_assign.append(tf.assign(var, avg))

    # Run all asign ops on an existing model (which has other ops and graph).
    sess = sessions[0]
    sess.run(all_assign)

    # Export a new saved model.
    ensure_dir_exists(FLAGS.data_dir)
    dest_path = os.path.join(FLAGS.data_dir, "swa-" + str(FLAGS.count))
    saver.save(sess, dest_path)
//  [python]
def swa():
    path_base = fsdb.models_dir()
    model_names = [
        "000393-lincoln",
        "000390-indus",
        "000404-hannibal",
        "000447-hawke",
        "000426-grief",
        "000431-lion",
        "000428-invincible",
        "000303-olympus",
        "000291-superb",
        "000454-victorious",
    ]
    model_names = model_names[:FLAGS.count]

    model_paths = [os.path.join(path_base, m) for m in model_names]

    # construct the graph
    features, labels = dual_net.get_inference_input()
    dual_net.model_fn(features, labels, tf.estimator.ModeKeys.PREDICT, FLAGS.flag_values_dict())

    # restore all saved weights
    meta_graph_def = meta_graph.read_meta_graph_file(model_paths[0] + '.meta')
    stored_var_names = set(
        [n.name for n in meta_graph_def.graph_def.node if n.op == 'VariableV2'])

    var_list = [v for v in tf.global_variables()
                if v.op.name in stored_var_names]
    var_list.sort(key=lambda v: v.op.name)

    print(stored_var_names)
    print(len(stored_var_names), len(var_list))

    sessions = [tf.Session() for _ in model_paths]
    saver = tf.train.Saver()
    for sess, model_path in zip(sessions, model_paths):
        saver.restore(sess, model_path)

    # Load all VariableV2s for each model.
    values = [sess.run(var_list) for sess in sessions]

    # Iterate over all variables average values from all models.
    all_assign = []
    for var, vals in zip(var_list, zip(*values)):
        print("{}x {}".format(len(vals), var))
        if var.name == "global_step:0":
            avg = vals[0]
            for val in vals:
                avg = tf.maximum(avg, val)
        else:
            avg = tf.add_n(vals) / len(vals)
            continue

        all_assign.append(tf.assign(var, avg))

    # Run all asign ops on an existing model (which has other ops and graph).
    sess = sessions[0]
    sess.run(all_assign)

    # Export a new saved model.
    ensure_dir_exists(FLAGS.data_dir)
    dest_path = os.path.join(FLAGS.data_dir, "swa-" + str(FLAGS.count))
    saver.save(sess, dest_path)
//  [python]
def resume(self, session):
		checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
		if checkpoint:
			if self.name and not self.name in checkpoint:
				print("IGNORING checkpoint of other run : " + checkpoint + " !")
				checkpoint = None
		else:
			print("NO checkpoint, nothing to resume")
		if checkpoint:
			print("LOADING " + checkpoint + " !")
			try:
				persister = tf.train.Saver(tf.all_variables())
				persister.restore(session, checkpoint)
				print("resume checkpoint successful!")
				return True
			except Exception as ex:
				print(ex)
				print("CANNOT LOAD checkpoint %s !" % checkpoint)
		return False
//  [python]
def resume(self, session):
		checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
		if checkpoint:
			if self.name and not self.name in checkpoint:
				print("IGNORING checkpoint of other run : " + checkpoint + " !")
				checkpoint = None
		else:
			print("NO checkpoint, nothing to resume")
		if checkpoint:
			print("LOADING " + checkpoint + " !")
			try:
				persister = tf.train.Saver(tf.global_variables())
				persister.restore(session, checkpoint)
				print("resume checkpoint successful!")
				return True
			except Exception as ex:
				print(ex)
				print("CANNOT LOAD checkpoint %s !" % checkpoint)
		return False
//  [python]
def __init__(self, model: str, cfg_file: str = None, *args, **kwargs):
        model = self.get_or_download_model_dir(model)
        tf.reset_default_graph()

        self.model_dir = model
        self.model_path = osp.join(model, ModelFile.TF_CHECKPOINT_FOLDER)
        if cfg_file is None:
            cfg_file = osp.join(model, ModelFile.CONFIGURATION)

        super().__init__(cfg_file)

        self.params = {}
        self._override_params_from_file()

        tf_config = tf.ConfigProto(allow_soft_placement=True)
        tf_config.gpu_options.allow_growth = True
        self._session = tf.Session(config=tf_config)

        self.source_wids = tf.placeholder(
            dtype=tf.int64, shape=[None, None], name='source_wids')
        self.target_wids = tf.placeholder(
            dtype=tf.int64, shape=[None, None], name='target_wids')
        self.output = {}

        self.global_step = tf.train.create_global_step()

        self.model = CsanmtForTranslation(self.model_path, **self.params)
        output = self.model(input=self.source_wids, label=self.target_wids)
        self.output.update(output)

        self.model_saver = tf.train.Saver(
            tf.all_variables(),
            max_to_keep=self.params['keep_checkpoint_max'])
        with self._session.as_default() as sess:
            logger.info(f'loading model from {self.model_path}')

            pretrained_variables_map = get_pretrained_variables_map(
                self.model_path)

            tf.train.init_from_checkpoint(self.model_path,
                                          pretrained_variables_map)
            sess.run(tf.global_variables_initializer())
//  [python]
def __init__(self, model: str, cfg_file: str = None, *args, **kwargs):
        model = self.get_or_download_model_dir(model)
        tf.reset_default_graph()

        self.model_dir = model
        self.model_path = osp.join(model, ModelFile.TF_CHECKPOINT_FOLDER)
        if cfg_file is None:
            cfg_file = osp.join(model, ModelFile.CONFIGURATION)

        super().__init__(cfg_file)

        self.params = {}
        self._override_params_from_file()

        tf_config = tf.ConfigProto(allow_soft_placement=True)
        tf_config.gpu_options.allow_growth = True
        self._session = tf.Session(config=tf_config)

        self.source_wids = tf.placeholder(
            dtype=tf.int64, shape=[None, None], name='source_wids')
        self.target_wids = tf.placeholder(
            dtype=tf.int64, shape=[None, None], name='target_wids')
        self.output = {}

        self.global_step = tf.train.create_global_step()

        self.model = CsanmtForTranslation(self.model_path, **self.params)
        output = self.model(input=self.source_wids, label=self.target_wids)
        self.output.update(output)

        self.model_saver = tf.train.Saver(
            tf.global_variables(),
            max_to_keep=self.params['keep_checkpoint_max'])
        with self._session.as_default() as sess:
            logger.info(f'loading model from {self.model_path}')

            pretrained_variables_map = get_pretrained_variables_map(
                self.model_path)

            tf.train.init_from_checkpoint(self.model_path,
                                          pretrained_variables_map)
            sess.run(tf.global_variables_initializer())
//  [python]
def get_pretrained_variables_map(checkpoint_file_path, ignore_scope=None):
    reader = tf.train.NewCheckpointReader(
        tf.train.latest_checkpoint(checkpoint_file_path))
    saved_shapes = reader.get_variable_to_shape_map()
    if ignore_scope is None:
        var_names = sorted([(var.name, var.name.split(':')[0])
                            for var in tf.all_variables()
                            if var.name.split(':')[0] in saved_shapes])
    else:
        var_names = sorted([(var.name, var.name.split(':')[0])
                            for var in tf.all_variables()
                            if var.name.split(':')[0] in saved_shapes and all(
                                scope not in var.name
                                for scope in ignore_scope)])
    restore_vars = []
    name2var = dict(
        zip(
            map(lambda x: x.name.split(':')[0], tf.all_variables()),
            tf.all_variables()))
    restore_map = {}
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = name2var[saved_var_name]
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
                restore_map[saved_var_name] = curr_var
    return restore_map
//  [python]
def get_pretrained_variables_map(checkpoint_file_path, ignore_scope=None):
    reader = tf.train.NewCheckpointReader(
        tf.train.latest_checkpoint(checkpoint_file_path))
    saved_shapes = reader.get_variable_to_shape_map()
    if ignore_scope is None:
        var_names = sorted([(var.name, var.name.split(':')[0])
                            for var in tf.global_variables()
                            if var.name.split(':')[0] in saved_shapes])
    else:
        var_names = sorted([(var.name, var.name.split(':')[0])
                            for var in tf.global_variables()
                            if var.name.split(':')[0] in saved_shapes and all(
                                scope not in var.name
                                for scope in ignore_scope)])
    restore_vars = []
    name2var = dict(
        zip(
            map(lambda x: x.name.split(':')[0], tf.global_variables()),
            tf.global_variables()))
    restore_map = {}
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = name2var[saved_var_name]
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
                restore_map[saved_var_name] = curr_var
    return restore_map
//  [python]
def train(self, data=0, steps=-1, dropout=None, display_step=10, test_step=100, batch_size=10,
	          resume=save_step):  # epochs=-1,
		print("learning_rate: %f" % self.learning_rate)
		if data: self.data = data
		steps = 9999999 if steps < 0 else steps
		session = self.session
		# with tf.device(_cpu):
		# t = tf.verify_tensor_all_finite(t, msg)
		tf.add_check_numerics_ops()
		self.summaries = tf.summary.merge_all()
		self.summary_writer = tf.summary.FileWriter(current_logdir(), session.graph)  
		if not dropout: dropout = 1.  # keep all
		x = self.x
		y = self.y
		keep_prob = self.keep_prob
		if not resume or not self.resume(session):
			session.run([tf.global_variables_initializer()])
		saver = tf.train.Saver(tf.all_variables())
		snapshot = self.name + str(get_last_tensorboard_run_nr())
		step = 0  # show first
		while step < steps:
			batch_xs, batch_ys = self.next_batch(batch_size, session)
			# batch_xs=np.array(batch_xs).reshape([-1]+self.input_shape)
			# print("step %d \r" % step)# end=' ')
			# tf.train.shuffle_batch_join(example_list, batch_size, capacity=min_queue_size + batch_size * 16, min_queue_size)
			# Fit training using batch data
			feed_dict = {x: batch_xs, y: batch_ys, keep_prob: dropout, self.train_phase: True}
			loss, _ = session.run([self.cost, self.optimize], feed_dict=feed_dict)
			if step % display_step == 0:
				seconds = int(time.time()) - start
				# Calculate batch accuracy, loss
				feed = {x: batch_xs, y: batch_ys, keep_prob: 1., self.train_phase: False}
				acc, summary = session.run([self.accuracy, self.summaries], feed_dict=feed)
				# self.summary_writer.add_summary(summary, step) # only test summaries for smoother curve
				print("\rStep {:d} Loss= {:.6f} Accuracy= {:.3f} Time= {:d}s".format(step, loss, acc, seconds), end=' ')
				if str(loss) == "nan": return print("\nLoss gradiant explosion, exiting!")  # restore!
			if step % test_step == 0: self.test(step)
			if step % save_step == 0 and step > 0:
				print("SAVING snapshot %s" % snapshot)
				saver.save(session, checkpoint_dir + "/" + snapshot + ".ckpt", self.global_step)

			step += 1
		print("\nOptimization Finished!")
		self.test(step, number=10000)
//  [python]
def train(self, data=0, steps=-1, dropout=None, display_step=10, test_step=100, batch_size=10,
	          resume=save_step):  # epochs=-1,
		print("learning_rate: %f" % self.learning_rate)
		if data: self.data = data
		steps = 9999999 if steps < 0 else steps
		session = self.session
		# with tf.device(_cpu):
		# t = tf.verify_tensor_all_finite(t, msg)
		tf.add_check_numerics_ops()
		self.summaries = tf.summary.merge_all()
		self.summary_writer = tf.summary.FileWriter(current_logdir(), session.graph)  
		if not dropout: dropout = 1.  # keep all
		x = self.x
		y = self.y
		keep_prob = self.keep_prob
		if not resume or not self.resume(session):
			session.run([tf.global_variables_initializer()])
		saver = tf.train.Saver(tf.global_variables())
		snapshot = self.name + str(get_last_tensorboard_run_nr())
		step = 0  # show first
		while step < steps:
			batch_xs, batch_ys = self.next_batch(batch_size, session)
			# batch_xs=np.array(batch_xs).reshape([-1]+self.input_shape)
			# print("step %d \r" % step)# end=' ')
			# tf.train.shuffle_batch_join(example_list, batch_size, capacity=min_queue_size + batch_size * 16, min_queue_size)
			# Fit training using batch data
			feed_dict = {x: batch_xs, y: batch_ys, keep_prob: dropout, self.train_phase: True}
			loss, _ = session.run([self.cost, self.optimize], feed_dict=feed_dict)
			if step % display_step == 0:
				seconds = int(time.time()) - start
				# Calculate batch accuracy, loss
				feed = {x: batch_xs, y: batch_ys, keep_prob: 1., self.train_phase: False}
				acc, summary = session.run([self.accuracy, self.summaries], feed_dict=feed)
				# self.summary_writer.add_summary(summary, step) # only test summaries for smoother curve
				print("\rStep {:d} Loss= {:.6f} Accuracy= {:.3f} Time= {:d}s".format(step, loss, acc, seconds), end=' ')
				if str(loss) == "nan": return print("\nLoss gradiant explosion, exiting!")  # restore!
			if step % test_step == 0: self.test(step)
			if step % save_step == 0 and step > 0:
				print("SAVING snapshot %s" % snapshot)
				saver.save(session, checkpoint_dir + "/" + snapshot + ".ckpt", self.global_step)

			step += 1
		print("\nOptimization Finished!")
		self.test(step, number=10000)
//  [python]
def _init_model(self, session):
        """Create Parser model and initialize with random or load parameters in session."""
        config_dict = load_config(self.model_config_path)
        config = get_config(config_dict, self.name)
        model_var_scope = get_model_var_scope(self.var_scope, self.name)
        print ("NOTICE: Initializing model var scope '%s'" % model_var_scope)
        # Check if self.model already exist
        if self.model is None:   # Create Graph Only once
            with tf.variable_scope(model_var_scope, reuse = tf.AUTO_REUSE):
                self.model = parse_model.NNParser(config=config)
        if len(glob.glob(self.ckpt_path + '.data*')) > 0: # file exist with pattern: 'parser.ckpt.data*'
            print("NOTICE: Loading model parameters from %s" % self.ckpt_path)
            all_vars = tf.all_variables()
            model_vars = [k for k in all_vars if model_var_scope in k.name.split("/")]   # Only Restore the Variable in Graph begin with parser/....
            tf.train.Saver(model_vars).restore(session, self.ckpt_path)
        else:
            print("NOTICE: Model not found, Try to run method: deepnlp.download('parse')")
            print("NOTICE: Created with fresh parameters.")
            session.run(tf.initialize_all_variables())
//  [python]
def _init_model(self, session):
        """Create Parser model and initialize with random or load parameters in session."""
        config_dict = load_config(self.model_config_path)
        config = get_config(config_dict, self.name)
        model_var_scope = get_model_var_scope(self.var_scope, self.name)
        print ("NOTICE: Initializing model var scope '%s'" % model_var_scope)
        # Check if self.model already exist
        if self.model is None:   # Create Graph Only once
            with tf.variable_scope(model_var_scope, reuse = tf.AUTO_REUSE):
                self.model = parse_model.NNParser(config=config)
        if len(glob.glob(self.ckpt_path + '.data*')) > 0: # file exist with pattern: 'parser.ckpt.data*'
            print("NOTICE: Loading model parameters from %s" % self.ckpt_path)
            all_vars = tf.global_variables()
            model_vars = [k for k in all_vars if model_var_scope in k.name.split("/")]   # Only Restore the Variable in Graph begin with parser/....
            tf.train.Saver(model_vars).restore(session, self.ckpt_path)
        else:
            print("NOTICE: Model not found, Try to run method: deepnlp.download('parse')")
            print("NOTICE: Created with fresh parameters.")
            session.run(tf.global_variables_initializer())
//  [python]
def initialize_uninitialized_variables(variables=None):
    if variables is None:
        variables = tf.global_variables()

    if not variables:
        return

    session = tensorflow_session()
    is_not_initialized = session.run([
        tf.is_variable_initialized(var) for var in variables])

    not_initialized_vars = [
        v for (v, f) in zip(variables, is_not_initialized) if not f]

    if len(not_initialized_vars):
        session.run(tf.variables_initializer(not_initialized_vars))
//  [python]
def restore_from_classification_checkpoint_fn(self, feature_extractor_scope):
    """Returns a map of variables to load from a foreign checkpoint.

    Args:
      feature_extractor_scope: A scope name for the feature extractor.

    Returns:
      A dict mapping variable names (to load from a checkpoint) to variables in
      the model graph.
    """
    variables_to_restore = {}
    for variable in tf.global_variables():
      var_name = variable.op.name
      if var_name.startswith(feature_extractor_scope + '/'):
        var_name = var_name.replace(feature_extractor_scope + '/', '')
        variables_to_restore[var_name] = variable

    return variables_to_restore
//  [python]
def evaluate():
    with tf.Graph().as_default():
        
        log_dir = '/home/kevin/tensorflow/CIFAR10/logs10000/'
        test_dir = '/home/kevin/tensorflow/CIFAR10/data/cifar-10-batches-bin/'
        n_test = 10000
        
        
        # reading test data
        images, labels = cifar10_input.read_cifar10(data_dir=test_dir,
                                                    is_train=False,
                                                    batch_size= BATCH_SIZE,
                                                    shuffle=False)

        logits = inference(images)
        top_k_op = tf.nn.in_top_k(logits, labels, 1)
        saver = tf.train.Saver(tf.all_variables())
        
        with tf.Session() as sess:
            
            print("Reading checkpoints...")
            ckpt = tf.train.get_checkpoint_state(log_dir)
            if ckpt and ckpt.model_checkpoint_path:
                global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
                saver.restore(sess, ckpt.model_checkpoint_path)
                print('Loading success, global_step is %s' % global_step)
            else:
                print('No checkpoint file found')
                return
        
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess = sess, coord = coord)
            
            try:
                num_iter = int(math.ceil(n_test / BATCH_SIZE))
                true_count = 0
                total_sample_count = num_iter * BATCH_SIZE
                step = 0

                while step < num_iter and not coord.should_stop():
                    predictions = sess.run([top_k_op])
                    true_count += np.sum(predictions)
                    step += 1
                    precision = true_count / total_sample_count
                print('precision = %.3f' % precision)
            except Exception as e:
                coord.request_stop(e)
            finally:
                coord.request_stop()
                coord.join(threads)
//  [python]
def evaluate():
    with tf.Graph().as_default():
        
        log_dir = '/home/kevin/tensorflow/CIFAR10/logs10000/'
        test_dir = '/home/kevin/tensorflow/CIFAR10/data/cifar-10-batches-bin/'
        n_test = 10000
        
        
        # reading test data
        images, labels = cifar10_input.read_cifar10(data_dir=test_dir,
                                                    is_train=False,
                                                    batch_size= BATCH_SIZE,
                                                    shuffle=False)

        logits = inference(images)
        top_k_op = tf.nn.in_top_k(logits, labels, 1)
        saver = tf.train.Saver(tf.global_variables())
        
        with tf.Session() as sess:
            
            print("Reading checkpoints...")
            ckpt = tf.train.get_checkpoint_state(log_dir)
            if ckpt and ckpt.model_checkpoint_path:
                global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
                saver.restore(sess, ckpt.model_checkpoint_path)
                print('Loading success, global_step is %s' % global_step)
            else:
                print('No checkpoint file found')
                return
        
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess = sess, coord = coord)
            
            try:
                num_iter = int(math.ceil(n_test / BATCH_SIZE))
                true_count = 0
                total_sample_count = num_iter * BATCH_SIZE
                step = 0

                while step < num_iter and not coord.should_stop():
                    predictions = sess.run([top_k_op])
                    true_count += np.sum(predictions)
                    step += 1
                    precision = true_count / total_sample_count
                print('precision = %.3f' % precision)
            except Exception as e:
                coord.request_stop(e)
            finally:
                coord.request_stop()
                coord.join(threads)
//  [python]
def testAccumulatedMomentsDuringTraing(self):
    with tf.Graph().as_default():
      mean_in = tf.placeholder(tf.float32, shape=[2])
      variance_in = tf.placeholder(tf.float32, shape=[2])
      mean, variance = arch_ops._accumulated_moments_for_inference(
          mean=mean_in, variance=variance_in, is_training=True)
      variables_by_name = {v.op.name: v for v in tf.all_variables()}
      tf.logging.error(variables_by_name)
      accu_mean = variables_by_name["accu/accu_mean"]
      accu_variance = variables_by_name["accu/accu_variance"]
      accu_counter = variables_by_name["accu/accu_counter"]
      with self.session() as sess:
        sess.run(tf.global_variables_initializer())
        m1, v1 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [1.0, 2.0], variance_in: [3.0, 4.0]})
        self.assertAllClose(m1, [1.0, 2.0])
        self.assertAllClose(v1, [3.0, 4.0])
        m2, v2 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [5.0, 6.0], variance_in: [7.0, 8.0]})
        self.assertAllClose(m2, [5.0, 6.0])
        self.assertAllClose(v2, [7.0, 8.0])
        am, av, ac = sess.run([accu_mean, accu_variance, accu_counter])
        self.assertAllClose(am, [0.0, 0.0])
        self.assertAllClose(av, [0.0, 0.0])
        self.assertAllClose([ac], [0.0])
//  [python]
def testAccumulatedMomentsDuringTraing(self):
    with tf.Graph().as_default():
      mean_in = tf.placeholder(tf.float32, shape=[2])
      variance_in = tf.placeholder(tf.float32, shape=[2])
      mean, variance = arch_ops._accumulated_moments_for_inference(
          mean=mean_in, variance=variance_in, is_training=True)
      variables_by_name = {v.op.name: v for v in tf.global_variables()}
      tf.logging.error(variables_by_name)
      accu_mean = variables_by_name["accu/accu_mean"]
      accu_variance = variables_by_name["accu/accu_variance"]
      accu_counter = variables_by_name["accu/accu_counter"]
      with self.session() as sess:
        sess.run(tf.global_variables_initializer())
        m1, v1 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [1.0, 2.0], variance_in: [3.0, 4.0]})
        self.assertAllClose(m1, [1.0, 2.0])
        self.assertAllClose(v1, [3.0, 4.0])
        m2, v2 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [5.0, 6.0], variance_in: [7.0, 8.0]})
        self.assertAllClose(m2, [5.0, 6.0])
        self.assertAllClose(v2, [7.0, 8.0])
        am, av, ac = sess.run([accu_mean, accu_variance, accu_counter])
        self.assertAllClose(am, [0.0, 0.0])
        self.assertAllClose(av, [0.0, 0.0])
        self.assertAllClose([ac], [0.0])
//  [python]
def restore_map(self,
                  fine_tune_checkpoint_type='detection',
                  load_all_detection_checkpoint_vars=False):
    """Returns a map of variables to load from a foreign checkpoint.

    See parent class for details.

    Args:
      fine_tune_checkpoint_type: whether to restore from a full detection
        checkpoint (with compatible variable names) or to restore from a
        classification checkpoint for initialization prior to training.
        Valid values: `detection`, `classification`. Default 'detection'.
      load_all_detection_checkpoint_vars: whether to load all variables (when
         `fine_tune_checkpoint_type='detection'`). If False, only variables
         within the appropriate scopes are included. Default False.

    Returns:
      A dict mapping variable names (to load from a checkpoint) to variables in
      the model graph.
    Raises:
      ValueError: if fine_tune_checkpoint_type is neither `classification`
        nor `detection`.
    """
    if fine_tune_checkpoint_type not in ['detection', 'classification']:
      raise ValueError('Not supported fine_tune_checkpoint_type: {}'.format(
          fine_tune_checkpoint_type))

    if fine_tune_checkpoint_type == 'classification':
      return self._feature_extractor.restore_from_classification_checkpoint_fn(
          self._extract_features_scope)

    if fine_tune_checkpoint_type == 'detection':
      variables_to_restore = {}
      for variable in tf.global_variables():
        var_name = variable.op.name
        if load_all_detection_checkpoint_vars:
          variables_to_restore[var_name] = variable
        else:
          if var_name.startswith(self._extract_features_scope):
            variables_to_restore[var_name] = variable

    return variables_to_restore
//  [python]
def testAccumulatedMomentsDuringEal(self):
    with tf.Graph().as_default():
      mean_in = tf.placeholder(tf.float32, shape=[2])
      variance_in = tf.placeholder(tf.float32, shape=[2])
      mean, variance = arch_ops._accumulated_moments_for_inference(
          mean=mean_in, variance=variance_in, is_training=False)
      variables_by_name = {v.op.name: v for v in tf.all_variables()}
      tf.logging.error(variables_by_name)
      accu_mean = variables_by_name["accu/accu_mean"]
      accu_variance = variables_by_name["accu/accu_variance"]
      accu_counter = variables_by_name["accu/accu_counter"]
      update_accus = variables_by_name["accu/update_accus"]
      with self.session() as sess:
        sess.run(tf.global_variables_initializer())
        # Fill accumulators.
        sess.run(tf.assign(update_accus, 1))
        m1, v1 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [1.0, 2.0], variance_in: [3.0, 4.0]})
        self.assertAllClose(m1, [1.0, 2.0])
        self.assertAllClose(v1, [3.0, 4.0])
        m2, v2 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [5.0, 6.0], variance_in: [7.0, 8.0]})
        self.assertAllClose(m2, [3.0, 4.0])
        self.assertAllClose(v2, [5.0, 6.0])
        # Check accumulators.
        am, av, ac = sess.run([accu_mean, accu_variance, accu_counter])
        self.assertAllClose(am, [6.0, 8.0])
        self.assertAllClose(av, [10.0, 12.0])
        self.assertAllClose([ac], [2.0])
        # Use accumulators.
        sess.run(tf.assign(update_accus, 0))
        m3, v3 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [2.0, 2.0], variance_in: [3.0, 3.0]})
        self.assertAllClose(m3, [3.0, 4.0])
        self.assertAllClose(v3, [5.0, 6.0])
        am, av, ac = sess.run([accu_mean, accu_variance, accu_counter])
        self.assertAllClose(am, [6.0, 8.0])
        self.assertAllClose(av, [10.0, 12.0])
        self.assertAllClose([ac], [2.0])
//  [python]
def testAccumulatedMomentsDuringEal(self):
    with tf.Graph().as_default():
      mean_in = tf.placeholder(tf.float32, shape=[2])
      variance_in = tf.placeholder(tf.float32, shape=[2])
      mean, variance = arch_ops._accumulated_moments_for_inference(
          mean=mean_in, variance=variance_in, is_training=False)
      variables_by_name = {v.op.name: v for v in tf.global_variables()}
      tf.logging.error(variables_by_name)
      accu_mean = variables_by_name["accu/accu_mean"]
      accu_variance = variables_by_name["accu/accu_variance"]
      accu_counter = variables_by_name["accu/accu_counter"]
      update_accus = variables_by_name["accu/update_accus"]
      with self.session() as sess:
        sess.run(tf.global_variables_initializer())
        # Fill accumulators.
        sess.run(tf.assign(update_accus, 1))
        m1, v1 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [1.0, 2.0], variance_in: [3.0, 4.0]})
        self.assertAllClose(m1, [1.0, 2.0])
        self.assertAllClose(v1, [3.0, 4.0])
        m2, v2 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [5.0, 6.0], variance_in: [7.0, 8.0]})
        self.assertAllClose(m2, [3.0, 4.0])
        self.assertAllClose(v2, [5.0, 6.0])
        # Check accumulators.
        am, av, ac = sess.run([accu_mean, accu_variance, accu_counter])
        self.assertAllClose(am, [6.0, 8.0])
        self.assertAllClose(av, [10.0, 12.0])
        self.assertAllClose([ac], [2.0])
        # Use accumulators.
        sess.run(tf.assign(update_accus, 0))
        m3, v3 = sess.run(
            [mean, variance],
            feed_dict={mean_in: [2.0, 2.0], variance_in: [3.0, 3.0]})
        self.assertAllClose(m3, [3.0, 4.0])
        self.assertAllClose(v3, [5.0, 6.0])
        am, av, ac = sess.run([accu_mean, accu_variance, accu_counter])
        self.assertAllClose(am, [6.0, 8.0])
        self.assertAllClose(av, [10.0, 12.0])
        self.assertAllClose([ac], [2.0])