jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def process_checkpoint(input_ckpt, output_ckpt_path, dense_layer):
    """
    This function loads a RN50 checkpoint with Dense layer as the final layer 
    and transforms the final dense layer into a 1x1 convolution layer. The weights
    of the dense layer are reshaped into weights of 1x1 conv layer.
    Args:
        input_ckpt: Path to the input RN50 ckpt which has dense layer as classification layer.
    Returns:
        None. New checkpoint with 1x1 conv layer as classification layer is generated.
    """
    with tf.Session() as sess:
        # Load all the variables
        all_vars = tf.train.list_variables(input_ckpt)
        # Capture the dense layer weights and reshape them to a 4D tensor which would be 
        # the weights of a 1x1 convolution layer. This code replaces the dense (FC) layer
        # to a 1x1 conv layer. 
        dense_layer_value=0.
        new_var_list=[]
        for var in all_vars:
            curr_var = tf.train.load_variable(input_ckpt, var[0])
            if var[0]==dense_layer:
                dense_layer_value = curr_var
            else:
                new_var_list.append(tf.Variable(curr_var, name=var[0]))
 
        dense_layer_shape = [1, 1, 2048, 1001]
        new_var_value = np.reshape(dense_layer_value, dense_layer_shape)
        new_var = tf.Variable(new_var_value, name=dense_layer)
        new_var_list.append(new_var)
        
        sess.run(tf.global_variables_initializer())
        tf.train.Saver(var_list=new_var_list).save(sess, output_ckpt_path, write_meta_graph=False, write_state=False)
        print ("Rewriting checkpoint completed")
//  [python]
def testEvaluation(self):
    batch_size = 2
    num_frames = 64
    height, width = 224, 224
    num_classes = 1000

    eval_inputs = tf.random_uniform((batch_size, num_frames, height, width, 3))
    logits, _ = s3dg.s3dg(eval_inputs, num_classes,
                          is_training=False)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def testEvaluation(self):
    batch_size = 2
    num_frames = 64
    height, width = 224, 224
    num_classes = 1000

    eval_inputs = tf.random_uniform((batch_size, num_frames, height, width, 3))
    logits, _ = s3dg.s3dg(eval_inputs, num_classes,
                          is_training=False)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def test_vocab():
    import tensorflow as tf
    import numpy as np
    import os
    from common_options import load_common_arguments

    os.environ["CUDA_VISIBLE_DEVICES"] = '0'
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    # Load global vocab
    args = load_common_arguments()
    global_vocab, global_vocab_size = load_vocab(args.global_vocab_file)

    vocab, vocab_size = load_vocab_dict(args.global_vocab_file)

    assert global_vocab_size == vocab_size

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        sess.run(tf.tables_initializer())
        i = 0
        ks = vocab.keys()
        vs = vocab.values()

        v1 = sess.run(global_vocab.lookup(tf.convert_to_tensor(ks)))
        for i in range(len(vs)):
            assert vs[i] == v1[i]
//  [python]
def test_vocab():
    import tensorflow as tf
    import numpy as np
    import os
    from common_options import load_common_arguments

    os.environ["CUDA_VISIBLE_DEVICES"] = '0'
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    # Load global vocab
    args = load_common_arguments()
    global_vocab, global_vocab_size = load_vocab(args.global_vocab_file)

    vocab, vocab_size = load_vocab_dict(args.global_vocab_file)

    assert global_vocab_size == vocab_size

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        sess.run(tf.tables_initializer())
        i = 0
        ks = vocab.keys()
        vs = vocab.values()

        v1 = sess.run(global_vocab.lookup(tf.convert_to_tensor(ks)))
        for i in range(len(vs)):
            assert vs[i] == v1[i]
//  [python]
def _convert_symbolic_tensor(symbolic_tensor) -> np.ndarray:
        import tensorflow as tf
        if tf.__version__ >= "2.0.0":
            with tf.compat.v1.Session() as sess:
                sess.run(tf.compat.v1.global_variables_initializer())
                tensor_as_np_array = sess.run(symbolic_tensor)
        else:
            # this is untested
            with tf.Session() as sess:
                sess.run(tf.initialize_all_variables())
                tensor_as_np_array = sess.run(symbolic_tensor)
        return tensor_as_np_array
//  [python]
def _convert_symbolic_tensor(symbolic_tensor) -> np.ndarray:
        import tensorflow as tf
        if tf.__version__ >= "2.0.0":
            with tf.compat.v1.Session() as sess:
                sess.run(tf.compat.v1.global_variables_initializer())
                tensor_as_np_array = sess.run(symbolic_tensor)
        else:
            # this is untested
            with tf.Session() as sess:
                sess.run(tf.global_variables_initializer())
                tensor_as_np_array = sess.run(symbolic_tensor)
        return tensor_as_np_array
//  [python]
def test():
    # below is a function test; if you use this for text classifiction, you need to tranform sentence to indices of vocabulary first. then feed data to the graph.
    num_classes = 9+2 #additional two classes:one is for _GO, another is for _END
    learning_rate = 0.0001
    batch_size = 1
    decay_steps = 1000
    decay_rate = 0.9
    sequence_length = 5
    vocab_size = 300
    embed_size = 100 #100
    hidden_size = 100
    is_training = True
    dropout_keep_prob = 1  # 0.5 #num_sentences
    decoder_sent_length=6
    l2_lambda=0.0001
    model = seq2seq_attention_model(num_classes, learning_rate, batch_size, decay_steps, decay_rate, sequence_length,
                                    vocab_size, embed_size,hidden_size, is_training,decoder_sent_length=decoder_sent_length,l2_lambda=l2_lambda)
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        for i in range(1500):
            #input_x = np.zeros((batch_size, sequence_length),dtype=np.int32) #[None, self.sequence_length]
            label_list=get_unique_labels()
            input_x = np.array([label_list],dtype=np.int32) #[2,3,4,5,6]
            label_list_original=copy.deepcopy(label_list)
            label_list.reverse()
            decoder_input=np.array([[0]+label_list],dtype=np.int32) #[[0,2,3,4,5,6]]
            input_y_label=np.array([label_list+[1]],dtype=np.int32) #[[2,3,4,5,6,1]]
            loss, acc, predict, W_projection_value, _ = sess.run([model.loss_val, model.accuracy, model.predictions, model.W_projection, model.train_op],
                                                     feed_dict={model.input_x:input_x,model.decoder_input:decoder_input, model.input_y_label: input_y_label,
                                                                model.dropout_keep_prob: dropout_keep_prob})
            print(i,"loss:", loss, "acc:", acc, "label_list_original as input x:",label_list_original,";input_y_label:", input_y_label, "prediction:", predict)
//  [python]
def test():
    # below is a function test; if you use this for text classifiction, you need to tranform sentence to indices of vocabulary first. then feed data to the graph.
    num_classes = 9+2 #additional two classes:one is for _GO, another is for _END
    learning_rate = 0.0001
    batch_size = 1
    decay_steps = 1000
    decay_rate = 0.9
    sequence_length = 5
    vocab_size = 300
    embed_size = 100 #100
    hidden_size = 100
    is_training = True
    dropout_keep_prob = 1  # 0.5 #num_sentences
    decoder_sent_length=6
    l2_lambda=0.0001
    model = seq2seq_attention_model(num_classes, learning_rate, batch_size, decay_steps, decay_rate, sequence_length,
                                    vocab_size, embed_size,hidden_size, is_training,decoder_sent_length=decoder_sent_length,l2_lambda=l2_lambda)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(1500):
            #input_x = np.zeros((batch_size, sequence_length),dtype=np.int32) #[None, self.sequence_length]
            label_list=get_unique_labels()
            input_x = np.array([label_list],dtype=np.int32) #[2,3,4,5,6]
            label_list_original=copy.deepcopy(label_list)
            label_list.reverse()
            decoder_input=np.array([[0]+label_list],dtype=np.int32) #[[0,2,3,4,5,6]]
            input_y_label=np.array([label_list+[1]],dtype=np.int32) #[[2,3,4,5,6,1]]
            loss, acc, predict, W_projection_value, _ = sess.run([model.loss_val, model.accuracy, model.predictions, model.W_projection, model.train_op],
                                                     feed_dict={model.input_x:input_x,model.decoder_input:decoder_input, model.input_y_label: input_y_label,
                                                                model.dropout_keep_prob: dropout_keep_prob})
            print(i,"loss:", loss, "acc:", acc, "label_list_original as input x:",label_list_original,";input_y_label:", input_y_label, "prediction:", predict)
//  [python]
def __init__(self,env,config):
        self.sess = tf.InteractiveSession()
        self.config = config

        self.replay_buffer = deque(maxlen = self.config.replay_buffer_size)
        self.time_step = 0

        self.state_dim = env.observation_space.shape
        self.action_dim = env.action_space.n

        print('state_dim:', self.state_dim)
        print('action_dim:', self.action_dim)

        self.action_batch = tf.placeholder('int32',[None])
        self.y_input = tf.placeholder('float',[None,self.action_dim])

        batch_shape = [None]
        batch_shape.extend(self.state_dim)

        self.eval_input = tf.placeholder('float',batch_shape)
        self.target_input = tf.placeholder('float',batch_shape)

        self.build_noisy_dqn_net()

        self.saver = tf.train.Saver()

        self.sess.run(tf.initialize_all_variables())

        self.save_model()
        self.restore_model()
//  [python]
def __init__(self,env,config):
        self.sess = tf.InteractiveSession()
        self.config = config

        self.replay_buffer = deque(maxlen = self.config.replay_buffer_size)
        self.time_step = 0

        self.state_dim = env.observation_space.shape
        self.action_dim = env.action_space.n

        print('state_dim:', self.state_dim)
        print('action_dim:', self.action_dim)

        self.action_batch = tf.placeholder('int32',[None])
        self.y_input = tf.placeholder('float',[None,self.action_dim])

        batch_shape = [None]
        batch_shape.extend(self.state_dim)

        self.eval_input = tf.placeholder('float',batch_shape)
        self.target_input = tf.placeholder('float',batch_shape)

        self.build_noisy_dqn_net()

        self.saver = tf.train.Saver()

        self.sess.run(tf.global_variables_initializer())

        self.save_model()
        self.restore_model()
//  [python]
def tf_baseline_conv2d():
    import tensorflow as tf
    import cntk.contrib.crosstalk.crosstalk_tensorflow as crtf
    ci = crtf.instance

    tf.reset_default_graph()

    x = tf.placeholder(tf.float32, [batch_size, num_chars, char_emb_dim])
    filter_bank = tf.get_variable("char_filter_bank",
                                  shape=[filter_width, char_emb_dim, num_filters],
                                  dtype=tf.float32)
    bias = tf.get_variable("char_filter_biases", shape=[num_filters], dtype=tf.float32)

    char_conv = tf.expand_dims(tf.transpose(tf.nn.conv1d(x, filter_bank, stride=1, padding='VALID') + bias, perm=[0,2,1]), -1)

    ci.watch(cstk.Conv2DArgs(W=crtf.find_trainable('char_filter_bank'), b=crtf.find_trainable('char_filter_biases')), 'conv2d', var_type=cstk.Conv2DAttr,
               attr=cstk.Conv2DAttr(filter_shape=(filter_width, char_emb_dim,), num_filters=num_filters))
    ci.watch(char_conv, 'conv2d_out', var_type=crtf.VariableType) # note the output is transposed to NCHW

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        data = {x:input_data}
        ci.set_workdir(workdir)
        ci.set_data(sess, data)
        ci.fetch('conv2d_out', save=True)
        ci.fetch('conv2d', save=True)
        ci.assign('conv2d', load=True)
        assert ci.compare('conv2d_out')
        ci.reset()
        sess.close()
//  [python]
def tf_baseline_conv2d():
    import tensorflow as tf
    import cntk.contrib.crosstalk.crosstalk_tensorflow as crtf
    ci = crtf.instance

    tf.reset_default_graph()

    x = tf.placeholder(tf.float32, [batch_size, num_chars, char_emb_dim])
    filter_bank = tf.get_variable("char_filter_bank",
                                  shape=[filter_width, char_emb_dim, num_filters],
                                  dtype=tf.float32)
    bias = tf.get_variable("char_filter_biases", shape=[num_filters], dtype=tf.float32)

    char_conv = tf.expand_dims(tf.transpose(tf.nn.conv1d(x, filter_bank, stride=1, padding='VALID') + bias, perm=[0,2,1]), -1)

    ci.watch(cstk.Conv2DArgs(W=crtf.find_trainable('char_filter_bank'), b=crtf.find_trainable('char_filter_biases')), 'conv2d', var_type=cstk.Conv2DAttr,
               attr=cstk.Conv2DAttr(filter_shape=(filter_width, char_emb_dim,), num_filters=num_filters))
    ci.watch(char_conv, 'conv2d_out', var_type=crtf.VariableType) # note the output is transposed to NCHW

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        data = {x:input_data}
        ci.set_workdir(workdir)
        ci.set_data(sess, data)
        ci.fetch('conv2d_out', save=True)
        ci.fetch('conv2d', save=True)
        ci.assign('conv2d', load=True)
        assert ci.compare('conv2d_out')
        ci.reset()
        sess.close()
//  [python]
def tf_baseline_embed():
    import tensorflow as tf
    import cntk.contrib.crosstalk.crosstalk_tensorflow as crtf
    ci = crtf.instance

    tf.reset_default_graph()
    
    embed = tf.get_variable("embed", initializer=emb1, dtype=tf.float32)
    ci.watch(embed, 'embed', var_type=cstk.EmbedAttr,
          attr=cstk.EmbedAttr(dict=dict1, input_dim=input_dim))

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        ci.set_workdir(workdir)
        ci.set_data(sess, None)
        ci.fetch('embed', save=True)
        ci.reset()
        sess.close()
//  [python]
def tf_baseline_embed():
    import tensorflow as tf
    import cntk.contrib.crosstalk.crosstalk_tensorflow as crtf
    ci = crtf.instance

    tf.reset_default_graph()
    
    embed = tf.get_variable("embed", initializer=emb1, dtype=tf.float32)
    ci.watch(embed, 'embed', var_type=cstk.EmbedAttr,
          attr=cstk.EmbedAttr(dict=dict1, input_dim=input_dim))

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        ci.set_workdir(workdir)
        ci.set_data(sess, None)
        ci.fetch('embed', save=True)
        ci.reset()
        sess.close()
//  [python]
def compareForwardExact(skip_theano=True, skip_tensorflow=False):
    """Decorates test methods, checking equality under multiple backends."""

    def decorator(test_func):

        def compare(self, *args):
            if not skip_theano:
                theano_result = test_func(self, th, *args).eval()
            if not skip_tensorflow:
                tf_session = tensorflow.Session()
                tf.set_session(tf_session)
                tensorflow_result_intermediate = test_func(self, tf, *args)
                tf_session.run(tf.global_variables_initializer())
                tensorflow_result = tensorflow_result_intermediate.eval(session=tf_session)
            plaidml_result = test_func(self, pkb, *args).eval()
            if not skip_theano:
                npt.assert_array_equal(plaidml_result,
                                       theano_result,
                                       err_msg='x=plaidml, y=theano')
            if not skip_tensorflow:
                npt.assert_array_equal(plaidml_result,
                                       tensorflow_result,
                                       err_msg='x=plaidml, y=tensorflow')
                tf_session.close()

        return compare

    return decorator
//  [python]
def compareForwardExact(skip_theano=True, skip_tensorflow=False):
    """Decorates test methods, checking equality under multiple backends."""

    def decorator(test_func):

        def compare(self, *args):
            if not skip_theano:
                theano_result = test_func(self, th, *args).eval()
            if not skip_tensorflow:
                tf_session = tensorflow.Session()
                tf.set_session(tf_session)
                tensorflow_result_intermediate = test_func(self, tf, *args)
                tf_session.run(tensorflow.global_variables_initializer())
                tensorflow_result = tensorflow_result_intermediate.eval(session=tf_session)
            plaidml_result = test_func(self, pkb, *args).eval()
            if not skip_theano:
                npt.assert_array_equal(plaidml_result,
                                       theano_result,
                                       err_msg='x=plaidml, y=theano')
            if not skip_tensorflow:
                npt.assert_array_equal(plaidml_result,
                                       tensorflow_result,
                                       err_msg='x=plaidml, y=tensorflow')
                tf_session.close()

        return compare

    return decorator
//  [python]
def create_session():
        config = tf.ConfigProto(allow_soft_placement=True)
        sess = tf.Session(config=config)
        sess.run(tf.initialize_all_variables())
        return sess
//  [python]
def create_session():
        config = tf.ConfigProto(allow_soft_placement=True)
        sess = tf.Session(config=config)
        sess.run(tf.global_variables_initializer())
        return sess
//  [python]
def compareForwardClose(epsilon=DEFAULT_TOL,
                        atol=DEFAULT_ATOL,
                        skip_theano=True,
                        skip_tensorflow=False):
    """Decorates test methods, checking near-equality under multiple backends."""

    def decorator(test_func):

        def compare(self, *args):
            if not skip_theano:
                theano_result = test_func(self, th, *args).eval()
            if not skip_tensorflow:
                tf_session = tensorflow.Session()
                tf.set_session(tf_session)
                tensorflow_result_intermediate = test_func(self, tf, *args)
                tf_session.run(tf.global_variables_initializer())
                tensorflow_result = tensorflow_result_intermediate.eval(session=tf_session)
            plaidml_result = test_func(self, pkb, *args).eval()
            if not skip_theano:
                npt.assert_allclose(plaidml_result,
                                    theano_result,
                                    rtol=epsilon,
                                    atol=atol,
                                    err_msg='x=plaidml, y=theano')
            if not skip_tensorflow:
                npt.assert_allclose(plaidml_result,
                                    tensorflow_result,
                                    rtol=epsilon,
                                    atol=atol,
                                    err_msg='x=plaidml, y=tensorflow')
                tf_session.close()

        return compare

    return decorator
//  [python]
def compareForwardClose(epsilon=DEFAULT_TOL,
                        atol=DEFAULT_ATOL,
                        skip_theano=True,
                        skip_tensorflow=False):
    """Decorates test methods, checking near-equality under multiple backends."""

    def decorator(test_func):

        def compare(self, *args):
            if not skip_theano:
                theano_result = test_func(self, th, *args).eval()
            if not skip_tensorflow:
                tf_session = tensorflow.Session()
                tf.set_session(tf_session)
                tensorflow_result_intermediate = test_func(self, tf, *args)
                tf_session.run(tensorflow.global_variables_initializer())
                tensorflow_result = tensorflow_result_intermediate.eval(session=tf_session)
            plaidml_result = test_func(self, pkb, *args).eval()
            if not skip_theano:
                npt.assert_allclose(plaidml_result,
                                    theano_result,
                                    rtol=epsilon,
                                    atol=atol,
                                    err_msg='x=plaidml, y=theano')
            if not skip_tensorflow:
                npt.assert_allclose(plaidml_result,
                                    tensorflow_result,
                                    rtol=epsilon,
                                    atol=atol,
                                    err_msg='x=plaidml, y=tensorflow')
                tf_session.close()

        return compare

    return decorator
//  [python]
def __init__(self, device, model_name, num_actions):
        self.device = device
        self.model_name = model_name
        self.num_actions = num_actions

        self.observation_size=Config.OBSERVATION_SIZE
        self.observation_channels = Config.STACKED_FRAMES

        self.learning_rate = Config.LEARNING_RATE_START
        self.beta = Config.BETA_START
        self.log_epsilon = Config.LOG_EPSILON

        self.graph = tf.Graph()
        with self.graph.as_default() as g:
            with tf.device(self.device):
                self._create_graph()

                self.sess = tf.Session(
                    graph=self.graph,
                    config=tf.ConfigProto(
                        allow_soft_placement=True,
                        log_device_placement=False,
                        gpu_options=tf.GPUOptions(allow_growth=True)))
                self.sess.run(tf.initialize_all_variables())

                if Config.TENSORBOARD: self._create_tensor_board()
                if Config.LOAD_CHECKPOINT or Config.SAVE_MODELS:
                    vars = tf.global_variables()
                    self.saver = tf.train.Saver({var.name: var for var in vars}, max_to_keep=0)
//  [python]
def __init__(self, device, model_name, num_actions):
        self.device = device
        self.model_name = model_name
        self.num_actions = num_actions

        self.observation_size=Config.OBSERVATION_SIZE
        self.observation_channels = Config.STACKED_FRAMES

        self.learning_rate = Config.LEARNING_RATE_START
        self.beta = Config.BETA_START
        self.log_epsilon = Config.LOG_EPSILON

        self.graph = tf.Graph()
        with self.graph.as_default() as g:
            with tf.device(self.device):
                self._create_graph()

                self.sess = tf.Session(
                    graph=self.graph,
                    config=tf.ConfigProto(
                        allow_soft_placement=True,
                        log_device_placement=False,
                        gpu_options=tf.GPUOptions(allow_growth=True)))
                self.sess.run(tf.global_variables_initializer())

                if Config.TENSORBOARD: self._create_tensor_board()
                if Config.LOAD_CHECKPOINT or Config.SAVE_MODELS:
                    vars = tf.global_variables()
                    self.saver = tf.train.Saver({var.name: var for var in vars}, max_to_keep=0)
//  [python]
def initialize_sessions(self) -> None:
        log("Initializing variables")
        init_op = tf.initialize_all_variables()
        init_tables = tf.tables_initializer()
        for sess in self.sessions:
            sess.run([init_op, init_tables])

        log("Initializing tf.train.Saver")
        self.saver = tf.train.Saver(max_to_keep=None,
                                    var_list=[g for g in tf.global_variables()
                                              if "reward_" not in g.name])
//  [python]
def initialize_sessions(self) -> None:
        log("Initializing variables")
        init_op = tf.global_variables_initializer()
        init_tables = tf.tables_initializer()
        for sess in self.sessions:
            sess.run([init_op, init_tables])

        log("Initializing tf.train.Saver")
        self.saver = tf.train.Saver(max_to_keep=None,
                                    var_list=[g for g in tf.global_variables()
                                              if "reward_" not in g.name])
//  [python]
def load(self):
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        embed = hub.Module("https://tfhub.dev/google/universal-sentence-encoder/2")
        session = tf.Session(config=config)
        session.run([tf.global_variables_initializer(), tf.tables_initializer()])
        input_placeholder = tf.placeholder(tf.string, shape=(None))
        embedded = embed(input_placeholder)
        self.model = lambda list_of_text: session.run(embedded, feed_dict={input_placeholder: list_of_text})
//  [python]
def train(args, train_data, test_data):
    model = DKN(args)

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        sess.run(tf.local_variables_initializer())

        for step in range(args.n_epochs):
            # training
            start_list = list(range(0, train_data.size, args.batch_size))
            np.random.shuffle(start_list)
            for start in start_list:
                end = start + args.batch_size
                model.train(sess, get_feed_dict(model, train_data, start, end))

            # evaluation
            train_auc = model.eval(sess, get_feed_dict(model, train_data, 0, train_data.size))
            test_auc = model.eval(sess, get_feed_dict(model, test_data, 0, test_data.size))
            print('epoch %d    train_auc: %.4f    test_auc: %.4f' % (step, train_auc, test_auc))
//  [python]
def train(args, train_data, test_data):
    model = DKN(args)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())

        for step in range(args.n_epochs):
            # training
            start_list = list(range(0, train_data.size, args.batch_size))
            np.random.shuffle(start_list)
            for start in start_list:
                end = start + args.batch_size
                model.train(sess, get_feed_dict(model, train_data, start, end))

            # evaluation
            train_auc = model.eval(sess, get_feed_dict(model, train_data, 0, train_data.size))
            test_auc = model.eval(sess, get_feed_dict(model, test_data, 0, test_data.size))
            print('epoch %d    train_auc: %.4f    test_auc: %.4f' % (step, train_auc, test_auc))
//  [python]
def load(self):
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        embed = hub.Module("https://tfhub.dev/google/universal-sentence-encoder-large/3")
        session = tf.Session(config=config)
        session.run([tf.global_variables_initializer(), tf.tables_initializer()])
        input_placeholder = tf.placeholder(tf.string, shape=(None))
        embedded = embed(input_placeholder)
        self.model = lambda list_of_text: session.run(embedded, feed_dict={input_placeholder: list_of_text})
//  [python]
def _train(self):
        
        self.num_batches = int(self.X_train.shape[0]//self.batch_size)
        self._build_model()
        self.saver = tf.train.Saver()
               
        
        with tf.Session() as sess:
            init = tf.initialize_all_variables()
            sess.run(init) 
            sess.run(self.assign_ops,feed_dict={self.emd_placeholder:self.embedding_matrix})            
            tf.train.write_graph(sess.graph_def, self.path, 'model.pbtxt') 
            print (self.batch_size,self.batch_size_val)
            for epoch in range(self.epochs):
                gen_batch = self.batch_gen(self.X_train,self.y_train,self.batch_size)
                gen_batch_val = self.batch_gen(self.X_val,self.y_val,self.batch_size_val)
                
                for batch in range(self.num_batches):
                    X_batch,y_batch = next(gen_batch) 
                    X_batch_val,y_batch_val = next(gen_batch_val)
                    sess.run(self.optimizer,feed_dict={self.X:X_batch,self.y:y_batch})
                    c,a  = sess.run([self.loss,self.accuracy],feed_dict={self.X:X_batch,self.y:y_batch})
                    print(" Epoch=",epoch," Batch=",batch," Training Loss: ","{:.9f}".format(c), " Training Accuracy=", "{:.9f}".format(a))
                    c1,a1 = sess.run([self.loss,self.accuracy],feed_dict={self.X:X_batch_val,self.y:y_batch_val})
                    print(" Epoch=",epoch," Validation Loss: ","{:.9f}".format(c1), " Validation Accuracy=", "{:.9f}".format(a1))
                results = sess.run(self.prediction_probability,feed_dict={self.X:X_batch_val})
                print(results)

                if epoch % self.checkpoint_step == 0:
                    self.saver.save(sess, os.path.join(self.path,'model'), global_step=epoch)    
            
            self.saver.save(sess,self.path + 'model_ckpt')
            results = sess.run(self.prediction_probability,feed_dict={self.X:X_batch_val})
            print(results)
//  [python]
def _train(self):
        
        self.num_batches = int(self.X_train.shape[0]//self.batch_size)
        self._build_model()
        self.saver = tf.train.Saver()
               
        
        with tf.Session() as sess:
            init = tf.global_variables_initializer()
            sess.run(init) 
            sess.run(self.assign_ops,feed_dict={self.emd_placeholder:self.embedding_matrix})            
            tf.train.write_graph(sess.graph_def, self.path, 'model.pbtxt') 
            print (self.batch_size,self.batch_size_val)
            for epoch in range(self.epochs):
                gen_batch = self.batch_gen(self.X_train,self.y_train,self.batch_size)
                gen_batch_val = self.batch_gen(self.X_val,self.y_val,self.batch_size_val)
                
                for batch in range(self.num_batches):
                    X_batch,y_batch = next(gen_batch) 
                    X_batch_val,y_batch_val = next(gen_batch_val)
                    sess.run(self.optimizer,feed_dict={self.X:X_batch,self.y:y_batch})
                    c,a  = sess.run([self.loss,self.accuracy],feed_dict={self.X:X_batch,self.y:y_batch})
                    print(" Epoch=",epoch," Batch=",batch," Training Loss: ","{:.9f}".format(c), " Training Accuracy=", "{:.9f}".format(a))
                    c1,a1 = sess.run([self.loss,self.accuracy],feed_dict={self.X:X_batch_val,self.y:y_batch_val})
                    print(" Epoch=",epoch," Validation Loss: ","{:.9f}".format(c1), " Validation Accuracy=", "{:.9f}".format(a1))
                results = sess.run(self.prediction_probability,feed_dict={self.X:X_batch_val})
                print(results)

                if epoch % self.checkpoint_step == 0:
                    self.saver.save(sess, os.path.join(self.path,'model'), global_step=epoch)    
            
            self.saver.save(sess,self.path + 'model_ckpt')
            results = sess.run(self.prediction_probability,feed_dict={self.X:X_batch_val})
            print(results)
//  [python]
def create_or_load_model(model, model_dir, session, name, ckpt_index=None):
  """Create translation model and initialize or load parameters in session."""
  if model_dir and ckpt_index is not None:
    ckpt_state = tf.train.get_checkpoint_state(model_dir)
    ckpt_path = ckpt_state.all_model_checkpoint_paths[ckpt_index]
  else:
    ckpt_path = tf.train.latest_checkpoint(model_dir)
  if ckpt_path:
    model = load_model(model, ckpt_path, session, name)
  else:
    start_time = time.time()
    session.run(tf.initialize_all_variables())
    session.run(tf.tables_initializer())
    utils.print_out("  created %s model with fresh parameters, time %.2fs" %
                    (name, time.time() - start_time))

  global_step = model.global_step.eval(session=session)
  return model, global_step
//  [python]
def create_or_load_model(model, model_dir, session, name, ckpt_index=None):
  """Create translation model and initialize or load parameters in session."""
  if model_dir and ckpt_index is not None:
    ckpt_state = tf.train.get_checkpoint_state(model_dir)
    ckpt_path = ckpt_state.all_model_checkpoint_paths[ckpt_index]
  else:
    ckpt_path = tf.train.latest_checkpoint(model_dir)
  if ckpt_path:
    model = load_model(model, ckpt_path, session, name)
  else:
    start_time = time.time()
    session.run(tf.global_variables_initializer())
    session.run(tf.tables_initializer())
    utils.print_out("  created %s model with fresh parameters, time %.2fs" %
                    (name, time.time() - start_time))

  global_step = model.global_step.eval(session=session)
  return model, global_step
//  [python]
def testUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 2
    height, width = 224, 224
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes)
      self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Conv2d_13_pointwise']
      feed_dict = {inputs: input_np}
      tf.initialize_all_variables().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024])
//  [python]
def testUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 2
    height, width = 224, 224
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes)
      self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Conv2d_13_pointwise']
      feed_dict = {inputs: input_np}
      tf.global_variables_initializer().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 7, 7, 1024])
//  [python]
def testEvaluation(self):
    batch_size = 2
    height, width = 224, 224
    num_classes = 1000

    eval_inputs = tf.random_uniform((batch_size, height, width, 3))
    logits, _ = mobilenet_v1.mobilenet_v1(eval_inputs, num_classes,
                                          is_training=False)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def testEvaluation(self):
    batch_size = 2
    height, width = 224, 224
    num_classes = 1000

    eval_inputs = tf.random_uniform((batch_size, height, width, 3))
    logits, _ = mobilenet_v1.mobilenet_v1(eval_inputs, num_classes,
                                          is_training=False)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (batch_size,))
//  [python]
def testGlobalPoolUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 1
    height, width = 250, 300
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes,
                                                     global_pool=True)
      self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Conv2d_13_pointwise']
      feed_dict = {inputs: input_np}
      tf.initialize_all_variables().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 10, 1024])
//  [python]
def testGlobalPoolUnknownImageShape(self):
    tf.reset_default_graph()
    batch_size = 1
    height, width = 250, 300
    num_classes = 1000
    input_np = np.random.uniform(0, 1, (batch_size, height, width, 3))
    with self.test_session() as sess:
      inputs = tf.placeholder(tf.float32, shape=(batch_size, None, None, 3))
      logits, end_points = mobilenet_v1.mobilenet_v1(inputs, num_classes,
                                                     global_pool=True)
      self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits'))
      self.assertListEqual(logits.get_shape().as_list(),
                           [batch_size, num_classes])
      pre_pool = end_points['Conv2d_13_pointwise']
      feed_dict = {inputs: input_np}
      tf.global_variables_initializer().run()
      pre_pool_out = sess.run(pre_pool, feed_dict=feed_dict)
      self.assertListEqual(list(pre_pool_out.shape), [batch_size, 8, 10, 1024])
//  [python]
def testUnknowBatchSize(self):
    batch_size = 1
    height, width = 224, 224
    num_classes = 1000

    inputs = tf.placeholder(tf.float32, (None, height, width, 3))
    logits, _ = mobilenet_v1.mobilenet_v1(inputs, num_classes)
    self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits'))
    self.assertListEqual(logits.get_shape().as_list(),
                         [None, num_classes])
    images = tf.random_uniform((batch_size, height, width, 3))

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch_size, num_classes))
//  [python]
def testUnknowBatchSize(self):
    batch_size = 1
    height, width = 224, 224
    num_classes = 1000

    inputs = tf.placeholder(tf.float32, (None, height, width, 3))
    logits, _ = mobilenet_v1.mobilenet_v1(inputs, num_classes)
    self.assertTrue(logits.op.name.startswith('MobilenetV1/Logits'))
    self.assertListEqual(logits.get_shape().as_list(),
                         [None, num_classes])
    images = tf.random_uniform((batch_size, height, width, 3))

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch_size, num_classes))
//  [python]
def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000

    train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
    mobilenet_v1.mobilenet_v1(train_inputs, num_classes)
    eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
    logits, _ = mobilenet_v1.mobilenet_v1(eval_inputs, num_classes,
                                          reuse=True)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,))
//  [python]
def testTrainEvalWithReuse(self):
    train_batch_size = 5
    eval_batch_size = 2
    height, width = 150, 150
    num_classes = 1000

    train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
    mobilenet_v1.mobilenet_v1(train_inputs, num_classes)
    eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
    logits, _ = mobilenet_v1.mobilenet_v1(eval_inputs, num_classes,
                                          reuse=True)
    predictions = tf.argmax(logits, 1)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(predictions)
      self.assertEquals(output.shape, (eval_batch_size,))
//  [python]
def testLogitsNotSqueezed(self):
    num_classes = 25
    images = tf.random_uniform([1, 224, 224, 3])
    logits, _ = mobilenet_v1.mobilenet_v1(images,
                                          num_classes=num_classes,
                                          spatial_squeeze=False)

    with self.test_session() as sess:
      tf.initialize_all_variables().run()
      logits_out = sess.run(logits)
      self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes])
//  [python]
def testLogitsNotSqueezed(self):
    num_classes = 25
    images = tf.random_uniform([1, 224, 224, 3])
    logits, _ = mobilenet_v1.mobilenet_v1(images,
                                          num_classes=num_classes,
                                          spatial_squeeze=False)

    with self.test_session() as sess:
      tf.global_variables_initializer().run()
      logits_out = sess.run(logits)
      self.assertListEqual(list(logits_out.shape), [1, 1, 1, num_classes])
//  [python]
def init_variables(self, YOLO_variables):
        init = tf.variables_initializer(YOLO_variables)
        # init = tf.global_variables_initializer()
        self.sess.run(init)
//  [python]
def init_variables(self, YOLO_variables):
        init = tf.global_variables_initializer()
        # init = tf.variables_initializer(YOLO_variables)
        self.sess.run(init)
//  [python]
def test_model_fn_train(self):
    with self.session() as sess:
      model_fn_builder = _get_model_builder()
      model_fn = model_fn_builder.build()
      output_spec = model_fn(
          self._features,
          labels=None,
          mode=tf.estimator.ModeKeys.TRAIN,
          params=None)

      sess.run([tf.global_variables_initializer(),
                tf.local_variables_initializer()])
      loss = sess.run(output_spec.loss)
      self.assertAllEqual(loss.shape, [])
//  [python]
def test_elastic_state(self):
        if not hvd._executing_eagerly() and _IS_TF2:
            # Only support TF 2.0 in eager mode
            return

        hvd.init()

        with tf.device("/cpu:0"):
            v = 1.0 if hvd.rank() == 0 else 2.0
            weights1 = [
                np.array([[v, v], [v, v]]),
                np.array([v, v])
            ]
            vars1 = [tf.Variable(arr) for arr in weights1]

            weights2 = [
                np.array([[1.0, 2.0], [3.0, 4.0]]),
                np.array([0.0, 0.0])
            ]

            if not hvd._executing_eagerly():
                init = tf.compat.v1.global_variables_initializer()
                self.evaluate(init)

            state = hvd.elastic.TensorFlowState(vars1, batch=20 + hvd.rank(), epoch=10 + hvd.rank())
            state.sync()

            weights1 = [np.ones_like(w) for w in weights1]

            # After sync, all values should match the root rank
            for w in self.evaluate(vars1):
                self.assertAllClose(w, np.ones_like(w))
            assert state.batch == 20
            assert state.epoch == 10

            # Partially modify then restore
            self.assign(vars1, weights2)
            state.batch = 21
            state.epoch = 11

            state.restore()

            for w1, w2 in zip(self.evaluate(vars1), weights1):
                self.assertAllClose(w1, w2)
            assert state.batch == 20
            assert state.epoch == 10

            # Partially modify then commit
            self.assign(vars1, weights2)
            state.batch = 21
            state.epoch = 11

            state.commit()
            state.restore()

            for w1, w2 in zip(self.evaluate(vars1), weights2):
                self.assertAllClose(w1, w2)
            assert state.batch == 21
            assert state.epoch == 11
//  [python]
def test_elastic_state(self):
        if not hvd._executing_eagerly() and _IS_TF2:
            # Only support TF 2.0 in eager mode
            return

        hvd.init()

        with tf.device("/cpu:0"):
            v = 1.0 if hvd.rank() == 0 else 2.0
            weights1 = [
                np.array([[v, v], [v, v]]),
                np.array([v, v])
            ]
            vars1 = [tf.Variable(arr) for arr in weights1]

            weights2 = [
                np.array([[1.0, 2.0], [3.0, 4.0]]),
                np.array([0.0, 0.0])
            ]

            if not hvd._executing_eagerly():
                init = tf.global_variables_initializer()
                self.evaluate(init)

            state = hvd.elastic.TensorFlowState(vars1, batch=20 + hvd.rank(), epoch=10 + hvd.rank())
            state.sync()

            weights1 = [np.ones_like(w) for w in weights1]

            # After sync, all values should match the root rank
            for w in self.evaluate(vars1):
                self.assertAllClose(w, np.ones_like(w))
            assert state.batch == 20
            assert state.epoch == 10

            # Partially modify then restore
            self.assign(vars1, weights2)
            state.batch = 21
            state.epoch = 11

            state.restore()

            for w1, w2 in zip(self.evaluate(vars1), weights1):
                self.assertAllClose(w1, w2)
            assert state.batch == 20
            assert state.epoch == 10

            # Partially modify then commit
            self.assign(vars1, weights2)
            state.batch = 21
            state.epoch = 11

            state.commit()
            state.restore()

            for w1, w2 in zip(self.evaluate(vars1), weights2):
                self.assertAllClose(w1, w2)
            assert state.batch == 21
            assert state.epoch == 11
//  [python]
def test_horovod_syncbn_gpu(self):
        """Test that the SyncBatchNormalization implementation is correct on GPU."""
        # Only do this test if there are GPUs available.
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest(("No GPUs available"))

        hvd.init()
        with tf.device("/gpu:%d" % hvd.local_rank()):
            x_list = [
                tf.convert_to_tensor(np.stack([
                    np.array([
                        [r, r + 1],
                        [r * 2, r * 2 + 1],
                        [r * 3, r * 3 + 1],
                        [r * 4, r * 4 + 1]
                    ], dtype=np.float32)
                    for r in range(hvd.size())
                ]), np.float32),
                tf.convert_to_tensor(np.stack([
                    np.array([
                        [r + 1],
                        [r * 2 + 1],
                        [r * 3 + 1],
                        [r * 4 + 1]
                    ], dtype=np.float32)
                    for r in range(hvd.size())
                ]), np.float32),
            ]

            for x in x_list:
                bn = tf.keras.layers.BatchNormalization(axis=1, fused=False)
                sync_bn = hvd.SyncBatchNormalization(axis=1)
                bn_func = bn(x, training=True)
                sync_bn_func = sync_bn(tf.expand_dims(x[hvd.rank()], 0), training=True)

                init = tf.compat.v1.global_variables_initializer()
                self.evaluate(init)
                bn_out = self.evaluate(bn_func)
                sync_bn_out = self.evaluate(sync_bn_func)

                self.assertAllClose(sync_bn_out, np.expand_dims(bn_out[hvd.rank()], 0))
                self.assertAllClose(self.evaluate(sync_bn.moving_mean), self.evaluate(bn.moving_mean))
                self.assertAllClose(self.evaluate(sync_bn.moving_variance), self.evaluate(bn.moving_variance))