jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def fit(self, X_list, Y, epochs = 1, callbacks = None, verbose = 1, batch_size = 64, shuffle = True):
        
        index = [ i for i in range(len(Y))]
        np.random.shuffle(index)
        X = X_list[0]
        Y_b = np.eye(self.num_classes)[Y]
        X = X[index]
        Y_b = Y_b[index]
        
        if batch_size >= len(X):
            batch_size = int(len(X)/2)
        
        train_step = tf.train.AdamOptimizer(0.0025).minimize(self.losses)
        init_global = tf.global_variables_initializer()
        init_local = tf.local_variables_initializer()
        saver = tf.train.Saver() 
        with tf.Session() as sess:
            sess.run([init_global, init_local])
            rounds = min(int(len(X)/batch_size), self.rr)

            for i in range(rounds):
                start = i*batch_size
                end = (i+1)*batch_size
                _ = sess.run(train_step, feed_dict = {self.input_x:X[start:end], self.input_y:Y_b[start:end]})
                
            self.rr += self.add_rr
            saver.save(sess, 'ft.ckpt')
            
        return 0
//  [python]
word = "Elephant"
sentence = "I am a sentence for which I would like to get its embedding."
paragraph = (
            "Universal Sentence Encoder embeddings also support short paragraphs. "
                "There is no hard limit on how long the paragraph is. Roughly, the longer "
                    "the more 'diluted' the embedding will be.")
messages = [word, sentence, paragraph]

with tf.Session() as session:
  session.run([tf.initialize_all_variables(), tf.tables_initializer()])
  message_embeddings = session.run(embed(messages))
 
  for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):
    print("Message: {}".format(messages[i]))
    print("Embedding size: {}".format(len(message_embedding)))
    message_embedding_snippet = ", ".join(
        (str(x) for x in message_embedding[:3]))
    print("Embedding: [{}, ...]\n".format(message_embedding_snippet))
//  [python]
word = "Elephant"
sentence = "I am a sentence for which I would like to get its embedding."
paragraph = (
            "Universal Sentence Encoder embeddings also support short paragraphs. "
                "There is no hard limit on how long the paragraph is. Roughly, the longer "
                    "the more 'diluted' the embedding will be.")
messages = [word, sentence, paragraph]

with tf.Session() as session:
  session.run([tf.global_variables_initializer(), tf.tables_initializer()])
  message_embeddings = session.run(embed(messages))
 
  for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):
    print("Message: {}".format(messages[i]))
    print("Embedding size: {}".format(len(message_embedding)))
    message_embedding_snippet = ", ".join(
        (str(x) for x in message_embedding[:3]))
    print("Embedding: [{}, ...]\n".format(message_embedding_snippet))
//  [python]
def fit_coin():
    X = []
    for line in open('coin_data.txt'):
        # 1 for H, 0 for T
        x = [1 if e == 'H' else 0 for e in line.rstrip()]
        X.append(x)

    # X = np.array(X).astype(np.int32)

    hmm = HMM(2)
    # the entire graph (including optimizer's variables) must be built
    # before calling global variables initializer!
    hmm.init_random(2)
    init = tf.initialize_all_variables()
    with tf.Session() as session:
        session.run(init)
        hmm.set_session(session)
        hmm.fit(X, max_iter=5)
        L = hmm.get_cost_multi(X).sum()
        print("LL with fitted params:", L)

        # try true values
        # remember these must be in their "pre-softmax" forms
        pi = np.log( np.array([0.5, 0.5]) ).astype(np.float32)
        A = np.log( np.array([[0.1, 0.9], [0.8, 0.2]]) ).astype(np.float32)
        B = np.log( np.array([[0.6, 0.4], [0.3, 0.7]]) ).astype(np.float32)
        hmm.set(pi, A, B)
        L = hmm.get_cost_multi(X).sum()
        print("LL with true params:", L)
//  [python]
def fit_coin():
    X = []
    for line in open('coin_data.txt'):
        # 1 for H, 0 for T
        x = [1 if e == 'H' else 0 for e in line.rstrip()]
        X.append(x)

    # X = np.array(X).astype(np.int32)

    hmm = HMM(2)
    # the entire graph (including optimizer's variables) must be built
    # before calling global variables initializer!
    hmm.init_random(2)
    init = tf.global_variables_initializer()
    with tf.Session() as session:
        session.run(init)
        hmm.set_session(session)
        hmm.fit(X, max_iter=5)
        L = hmm.get_cost_multi(X).sum()
        print("LL with fitted params:", L)

        # try true values
        # remember these must be in their "pre-softmax" forms
        pi = np.log( np.array([0.5, 0.5]) ).astype(np.float32)
        A = np.log( np.array([[0.1, 0.9], [0.8, 0.2]]) ).astype(np.float32)
        B = np.log( np.array([[0.6, 0.4], [0.3, 0.7]]) ).astype(np.float32)
        hmm.set(pi, A, B)
        L = hmm.get_cost_multi(X).sum()
        print("LL with true params:", L)
//  [python]
def verify_fused_batch_norm(shape):
    g = tf.Graph()
    with g.as_default():
        input_tensor = tf.placeholder(tf.float32, shape=shape, name="input")
        alpha = tf.constant(
            np.random.rand(
                shape[-1],
            ),
            dtype=tf.float32,
            name="alpha",
        )
        beta = tf.constant(
            np.random.rand(
                shape[-1],
            ),
            dtype=tf.float32,
            name="beta",
        )
        bn = tf.nn.fused_batch_norm(x=input_tensor, offset=beta, scale=alpha, name="bn")
        out = tf.identity(bn[0], name="output")
    data = np.random.rand(*shape)
    with tf.Session(graph=out.graph) as sess:
        sess.run(tf.compat.v1.global_variables_initializer())
        tf_out = sess.run(out, feed_dict={input_tensor: data})
        constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, ["output"])

    for device in ["llvm"]:
        dev = tvm.device(device, 0)
        if not tvm.testing.device_enabled(device):
            print("Skip because %s is not enabled" % device)
            continue
        with tvm.testing.disable_span_filling():
            mod, params = relay.frontend.from_tensorflow(constant_graph, outputs=["output"])
        with tvm.testing.enable_span_filling():
            mod_with_span, _ = relay.frontend.from_tensorflow(constant_graph, outputs=["output"])
        assert tvm.ir.structural_equal(mod["main"], mod_with_span["main"])
        with tvm.transform.PassContext(opt_level=3):
            graph, lib, params = relay.build(mod, target=device, params=params)
        from tvm.contrib import graph_executor

        m = graph_executor.create(graph, lib, dev)
        m.set_input(**params)
        m.set_input("input", data)
        m.run()
        tvm_out = m.get_output(0)
        tvm.testing.assert_allclose(
            tvm_out.numpy(), tf_out.astype(tvm_out.dtype), atol=1e-3, rtol=1e-3
        )
//  [python]
def verify_fused_batch_norm(shape):
    g = tf.Graph()
    with g.as_default():
        input_tensor = tf.placeholder(tf.float32, shape=shape, name="input")
        alpha = tf.constant(
            np.random.rand(
                shape[-1],
            ),
            dtype=tf.float32,
            name="alpha",
        )
        beta = tf.constant(
            np.random.rand(
                shape[-1],
            ),
            dtype=tf.float32,
            name="beta",
        )
        bn = tf.nn.fused_batch_norm(x=input_tensor, offset=beta, scale=alpha, name="bn")
        out = tf.identity(bn[0], name="output")
    data = np.random.rand(*shape)
    with tf.Session(graph=out.graph) as sess:
        sess.run([tf.global_variables_initializer()])
        tf_out = sess.run(out, feed_dict={input_tensor: data})
        constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, ["output"])

    for device in ["llvm"]:
        dev = tvm.device(device, 0)
        if not tvm.testing.device_enabled(device):
            print("Skip because %s is not enabled" % device)
            continue
        with tvm.testing.disable_span_filling():
            mod, params = relay.frontend.from_tensorflow(constant_graph, outputs=["output"])
        with tvm.testing.enable_span_filling():
            mod_with_span, _ = relay.frontend.from_tensorflow(constant_graph, outputs=["output"])
        assert tvm.ir.structural_equal(mod["main"], mod_with_span["main"])
        with tvm.transform.PassContext(opt_level=3):
            graph, lib, params = relay.build(mod, target=device, params=params)
        from tvm.contrib import graph_executor

        m = graph_executor.create(graph, lib, dev)
        m.set_input(**params)
        m.set_input("input", data)
        m.run()
        tvm_out = m.get_output(0)
        tvm.testing.assert_allclose(
            tvm_out.numpy(), tf_out.astype(tvm_out.dtype), atol=1e-3, rtol=1e-3
        )
//  [python]
def test_MpiAdam():
    np.random.seed(0)
    tf.set_random_seed(0)

    a = tf.Variable(np.random.randn(3).astype('float32'))
    b = tf.Variable(np.random.randn(2,5).astype('float32'))
    loss = tf.reduce_sum(tf.square(a)) + tf.reduce_sum(tf.sin(b))

    stepsize = 1e-2
    update_op = tf.train.AdamOptimizer(stepsize).minimize(loss)
    do_update = U.function([], loss, updates=[update_op])

    tf.get_default_session().run(tf.global_variables_initializer())
    losslist_ref = []
    for i in range(10):
        l = do_update()
        print(i, l)
        losslist_ref.append(l)



    tf.set_random_seed(0)
    tf.get_default_session().run(tf.global_variables_initializer())

    var_list = [a,b]
    lossandgrad = U.function([], [loss, U.flatgrad(loss, var_list)])
    adam = MpiAdam(var_list)

    losslist_test = []
    for i in range(10):
        l,g = lossandgrad()
        adam.update(g, stepsize)
        print(i,l)
        losslist_test.append(l)

    np.testing.assert_allclose(np.array(losslist_ref), np.array(losslist_test), atol=1e-4)
//  [python]
def load_model(self, model: str, model_path: str, max_seq_length: int):
        g = tf.Graph()
        with g.as_default():
            self.input_ids = tf.placeholder(dtype=tf.int32, shape=[None, max_seq_length])
            self.input_masks = tf.placeholder(dtype=tf.int32, shape=[None, max_seq_length])
            self.segment_ids = tf.placeholder(dtype=tf.int32, shape=[None, max_seq_length])

            hub_module = hub.Module(model_path)
            albert_inputs = dict(
                input_ids=self.input_ids,
                input_mask=self.input_masks,
                segment_ids=self.segment_ids
            )
            self.albert_outputs = hub_module(albert_inputs, signature="tokens", as_dict=True)
            tokenization_info = hub_module(signature="tokenization_info", as_dict=True)
            init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])
        g.finalize()
        self.sess = tf.Session(graph=g)
        self.sess.run(init_op)
        sentence_piece_file, do_lower_case = self.sess.run(
            [tokenization_info["vocab_file"], tokenization_info["do_lower_case"]]
        )
        Embeddings.tokenizer = FullTokenizer(vocab_file=None,
                                             do_lower_case=do_lower_case,
                                             spm_model_file=sentence_piece_file)
        self.max_seq_length = max_seq_length
        self.model_name = model
        print("Model loaded Successfully !")
//  [python]
def test_integration(self):
        """Test the values."""
        with tf.Session(graph=tf.Graph()) as sess:
            a = np.array([[1.2]])
            b = np.array([[0.9]])
            q = np.array([[1]])
            r = np.array([[0.1]])

            k, p = dlqr(a, b, q, r)
            true_value = QuadraticFunction(-p)

            discretization = GridWorld([[-1, 1]], 19)
            value_function = Triangulation(discretization,
                                           0. * discretization.all_points,
                                           project=True)

            dynamics = LinearSystem((a, b))

            policy_discretization = GridWorld([-1, 1], 5)
            policy = Triangulation(policy_discretization,
                                   -k / 2 * policy_discretization.all_points)
            reward_function = QuadraticFunction(-scipy.linalg.block_diag(q, r))

            rl = PolicyIteration(policy,
                                 dynamics,
                                 reward_function,
                                 value_function)

            value_iter = rl.value_iteration()

            loss = -tf.reduce_sum(rl.future_values(rl.state_space))
            optimizer = tf.train.GradientDescentOptimizer(0.01)
            adapt_policy = optimizer.minimize(loss,
                                              var_list=rl.policy.parameters)

            sess.run(tf.initialize_all_variables())

            for _ in range(10):
                sess.run(value_iter)
                for _ in range(5):
                    sess.run(adapt_policy)

            values = rl.value_function.parameters[0].eval()
            true_values = true_value(rl.state_space).eval()
            policy_values = rl.policy.parameters[0].eval()

        assert_allclose(values, true_values, atol=0.1)
        assert_allclose(policy_values, -k * policy_discretization.all_points,
                        atol=0.1)
//  [python]
def test_integration(self):
        """Test the values."""
        with tf.Session(graph=tf.Graph()) as sess:
            a = np.array([[1.2]])
            b = np.array([[0.9]])
            q = np.array([[1]])
            r = np.array([[0.1]])

            k, p = dlqr(a, b, q, r)
            true_value = QuadraticFunction(-p)

            discretization = GridWorld([[-1, 1]], 19)
            value_function = Triangulation(discretization,
                                           0. * discretization.all_points,
                                           project=True)

            dynamics = LinearSystem((a, b))

            policy_discretization = GridWorld([-1, 1], 5)
            policy = Triangulation(policy_discretization,
                                   -k / 2 * policy_discretization.all_points)
            reward_function = QuadraticFunction(-scipy.linalg.block_diag(q, r))

            rl = PolicyIteration(policy,
                                 dynamics,
                                 reward_function,
                                 value_function)

            value_iter = rl.value_iteration()

            loss = -tf.reduce_sum(rl.future_values(rl.state_space))
            optimizer = tf.train.GradientDescentOptimizer(0.01)
            adapt_policy = optimizer.minimize(loss,
                                              var_list=rl.policy.parameters)

            sess.run(tf.global_variables_initializer())

            for _ in range(10):
                sess.run(value_iter)
                for _ in range(5):
                    sess.run(adapt_policy)

            values = rl.value_function.parameters[0].eval()
            true_values = true_value(rl.state_space).eval()
            policy_values = rl.policy.parameters[0].eval()

        assert_allclose(values, true_values, atol=0.1)
        assert_allclose(policy_values, -k * policy_discretization.all_points,
                        atol=0.1)
//  [python]
def test_get_expected_feature_map_shapes_with_inception_v2(self, use_keras):
    image_features = {
        'Mixed_3c': tf.random_uniform([4, 28, 28, 256], dtype=tf.float32),
        'Mixed_4c': tf.random_uniform([4, 14, 14, 576], dtype=tf.float32),
        'Mixed_5c': tf.random_uniform([4, 7, 7, 1024], dtype=tf.float32)
    }
    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=INCEPTION_V2_LAYOUT,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Mixed_3c': (4, 28, 28, 256),
        'Mixed_4c': (4, 14, 14, 576),
        'Mixed_5c': (4, 7, 7, 1024),
        'Mixed_5c_2_Conv2d_3_3x3_s2_512': (4, 4, 4, 512),
        'Mixed_5c_2_Conv2d_4_3x3_s2_256': (4, 2, 2, 256),
        'Mixed_5c_2_Conv2d_5_3x3_s2_256': (4, 1, 1, 256)}

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_inception_v2(self, use_keras):
    image_features = {
        'Mixed_3c': tf.random_uniform([4, 28, 28, 256], dtype=tf.float32),
        'Mixed_4c': tf.random_uniform([4, 14, 14, 576], dtype=tf.float32),
        'Mixed_5c': tf.random_uniform([4, 7, 7, 1024], dtype=tf.float32)
    }
    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=INCEPTION_V2_LAYOUT,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Mixed_3c': (4, 28, 28, 256),
        'Mixed_4c': (4, 14, 14, 576),
        'Mixed_5c': (4, 7, 7, 1024),
        'Mixed_5c_2_Conv2d_3_3x3_s2_512': (4, 4, 4, 512),
        'Mixed_5c_2_Conv2d_4_3x3_s2_256': (4, 2, 2, 256),
        'Mixed_5c_2_Conv2d_5_3x3_s2_256': (4, 1, 1, 256)}

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_use_explicit_padding(
      self, use_keras):
    image_features = {
        'Mixed_3c': tf.random_uniform([4, 28, 28, 256], dtype=tf.float32),
        'Mixed_4c': tf.random_uniform([4, 14, 14, 576], dtype=tf.float32),
        'Mixed_5c': tf.random_uniform([4, 7, 7, 1024], dtype=tf.float32)
    }
    layout_copy = INCEPTION_V2_LAYOUT.copy()
    layout_copy['use_explicit_padding'] = True
    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=layout_copy,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Mixed_3c': (4, 28, 28, 256),
        'Mixed_4c': (4, 14, 14, 576),
        'Mixed_5c': (4, 7, 7, 1024),
        'Mixed_5c_2_Conv2d_3_3x3_s2_512': (4, 4, 4, 512),
        'Mixed_5c_2_Conv2d_4_3x3_s2_256': (4, 2, 2, 256),
        'Mixed_5c_2_Conv2d_5_3x3_s2_256': (4, 1, 1, 256)}

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_use_explicit_padding(
      self, use_keras):
    image_features = {
        'Mixed_3c': tf.random_uniform([4, 28, 28, 256], dtype=tf.float32),
        'Mixed_4c': tf.random_uniform([4, 14, 14, 576], dtype=tf.float32),
        'Mixed_5c': tf.random_uniform([4, 7, 7, 1024], dtype=tf.float32)
    }
    layout_copy = INCEPTION_V2_LAYOUT.copy()
    layout_copy['use_explicit_padding'] = True
    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=layout_copy,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Mixed_3c': (4, 28, 28, 256),
        'Mixed_4c': (4, 14, 14, 576),
        'Mixed_5c': (4, 7, 7, 1024),
        'Mixed_5c_2_Conv2d_3_3x3_s2_512': (4, 4, 4, 512),
        'Mixed_5c_2_Conv2d_4_3x3_s2_256': (4, 2, 2, 256),
        'Mixed_5c_2_Conv2d_5_3x3_s2_256': (4, 1, 1, 256)}

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_inception_v3(self, use_keras):
    image_features = {
        'Mixed_5d': tf.random_uniform([4, 35, 35, 256], dtype=tf.float32),
        'Mixed_6e': tf.random_uniform([4, 17, 17, 576], dtype=tf.float32),
        'Mixed_7c': tf.random_uniform([4, 8, 8, 1024], dtype=tf.float32)
    }

    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=INCEPTION_V3_LAYOUT,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Mixed_5d': (4, 35, 35, 256),
        'Mixed_6e': (4, 17, 17, 576),
        'Mixed_7c': (4, 8, 8, 1024),
        'Mixed_7c_2_Conv2d_3_3x3_s2_512': (4, 4, 4, 512),
        'Mixed_7c_2_Conv2d_4_3x3_s2_256': (4, 2, 2, 256),
        'Mixed_7c_2_Conv2d_5_3x3_s2_128': (4, 1, 1, 128)}

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_inception_v3(self, use_keras):
    image_features = {
        'Mixed_5d': tf.random_uniform([4, 35, 35, 256], dtype=tf.float32),
        'Mixed_6e': tf.random_uniform([4, 17, 17, 576], dtype=tf.float32),
        'Mixed_7c': tf.random_uniform([4, 8, 8, 1024], dtype=tf.float32)
    }

    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=INCEPTION_V3_LAYOUT,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Mixed_5d': (4, 35, 35, 256),
        'Mixed_6e': (4, 17, 17, 576),
        'Mixed_7c': (4, 8, 8, 1024),
        'Mixed_7c_2_Conv2d_3_3x3_s2_512': (4, 4, 4, 512),
        'Mixed_7c_2_Conv2d_4_3x3_s2_256': (4, 2, 2, 256),
        'Mixed_7c_2_Conv2d_5_3x3_s2_128': (4, 1, 1, 128)}

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_embedded_ssd_mobilenet_v1(
      self, use_keras):
    image_features = {
        'Conv2d_11_pointwise': tf.random_uniform([4, 16, 16, 512],
                                                 dtype=tf.float32),
        'Conv2d_13_pointwise': tf.random_uniform([4, 8, 8, 1024],
                                                 dtype=tf.float32),
    }

    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=EMBEDDED_SSD_MOBILENET_V1_LAYOUT,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Conv2d_11_pointwise': (4, 16, 16, 512),
        'Conv2d_13_pointwise': (4, 8, 8, 1024),
        'Conv2d_13_pointwise_2_Conv2d_2_3x3_s2_512': (4, 4, 4, 512),
        'Conv2d_13_pointwise_2_Conv2d_3_3x3_s2_256': (4, 2, 2, 256),
        'Conv2d_13_pointwise_2_Conv2d_4_2x2_s2_256': (4, 1, 1, 256)}

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_embedded_ssd_mobilenet_v1(
      self, use_keras):
    image_features = {
        'Conv2d_11_pointwise': tf.random_uniform([4, 16, 16, 512],
                                                 dtype=tf.float32),
        'Conv2d_13_pointwise': tf.random_uniform([4, 8, 8, 1024],
                                                 dtype=tf.float32),
    }

    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=EMBEDDED_SSD_MOBILENET_V1_LAYOUT,
        use_keras=use_keras
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Conv2d_11_pointwise': (4, 16, 16, 512),
        'Conv2d_13_pointwise': (4, 8, 8, 1024),
        'Conv2d_13_pointwise_2_Conv2d_2_3x3_s2_512': (4, 4, 4, 512),
        'Conv2d_13_pointwise_2_Conv2d_3_3x3_s2_256': (4, 2, 2, 256),
        'Conv2d_13_pointwise_2_Conv2d_4_2x2_s2_256': (4, 1, 1, 256)}

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_feature_map_shapes_with_pool_residual_ssd_mobilenet_v1(
      self, use_keras):
    image_features = {
        'Conv2d_13_pointwise': tf.random_uniform([4, 8, 8, 1024],
                                                 dtype=tf.float32),
    }

    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=SSD_MOBILENET_V1_WEIGHT_SHARED_LAYOUT,
        use_keras=use_keras,
        pool_residual=True
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Conv2d_13_pointwise': (4, 8, 8, 1024),
        'Conv2d_13_pointwise_2_Conv2d_1_3x3_s2_256': (4, 4, 4, 256),
        'Conv2d_13_pointwise_2_Conv2d_2_3x3_s2_256': (4, 2, 2, 256),
        'Conv2d_13_pointwise_2_Conv2d_3_3x3_s2_256': (4, 1, 1, 256)}

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_feature_map_shapes_with_pool_residual_ssd_mobilenet_v1(
      self, use_keras):
    image_features = {
        'Conv2d_13_pointwise': tf.random_uniform([4, 8, 8, 1024],
                                                 dtype=tf.float32),
    }

    feature_map_generator = self._build_feature_map_generator(
        feature_map_layout=SSD_MOBILENET_V1_WEIGHT_SHARED_LAYOUT,
        use_keras=use_keras,
        pool_residual=True
    )
    feature_maps = feature_map_generator(image_features)

    expected_feature_map_shapes = {
        'Conv2d_13_pointwise': (4, 8, 8, 1024),
        'Conv2d_13_pointwise_2_Conv2d_1_3x3_s2_256': (4, 4, 4, 256),
        'Conv2d_13_pointwise_2_Conv2d_2_3x3_s2_256': (4, 2, 2, 256),
        'Conv2d_13_pointwise_2_Conv2d_3_3x3_s2_256': (4, 1, 1, 256)}

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = dict(
          (key, value.shape) for key, value in out_feature_maps.items())
      self.assertDictEqual(expected_feature_map_shapes, out_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes(self):
    image_features = [
        ('block2', tf.random_uniform([4, 8, 8, 256], dtype=tf.float32)),
        ('block3', tf.random_uniform([4, 4, 4, 256], dtype=tf.float32)),
        ('block4', tf.random_uniform([4, 2, 2, 256], dtype=tf.float32)),
        ('block5', tf.random_uniform([4, 1, 1, 256], dtype=tf.float32))
    ]
    feature_maps = feature_map_generators.fpn_top_down_feature_maps(
        image_features=image_features, depth=128)

    expected_feature_map_shapes = {
        'top_down_block2': (4, 8, 8, 128),
        'top_down_block3': (4, 4, 4, 128),
        'top_down_block4': (4, 2, 2, 128),
        'top_down_block5': (4, 1, 1, 128)
    }

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = {key: value.shape
                                for key, value in out_feature_maps.items()}
      self.assertDictEqual(out_feature_map_shapes, expected_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes(self):
    image_features = [
        ('block2', tf.random_uniform([4, 8, 8, 256], dtype=tf.float32)),
        ('block3', tf.random_uniform([4, 4, 4, 256], dtype=tf.float32)),
        ('block4', tf.random_uniform([4, 2, 2, 256], dtype=tf.float32)),
        ('block5', tf.random_uniform([4, 1, 1, 256], dtype=tf.float32))
    ]
    feature_maps = feature_map_generators.fpn_top_down_feature_maps(
        image_features=image_features, depth=128)

    expected_feature_map_shapes = {
        'top_down_block2': (4, 8, 8, 128),
        'top_down_block3': (4, 4, 4, 128),
        'top_down_block4': (4, 2, 2, 128),
        'top_down_block5': (4, 1, 1, 128)
    }

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = {key: value.shape
                                for key, value in out_feature_maps.items()}
      self.assertDictEqual(out_feature_map_shapes, expected_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_depthwise(self):
    image_features = [
        ('block2', tf.random_uniform([4, 8, 8, 256], dtype=tf.float32)),
        ('block3', tf.random_uniform([4, 4, 4, 256], dtype=tf.float32)),
        ('block4', tf.random_uniform([4, 2, 2, 256], dtype=tf.float32)),
        ('block5', tf.random_uniform([4, 1, 1, 256], dtype=tf.float32))
    ]
    feature_maps = feature_map_generators.fpn_top_down_feature_maps(
        image_features=image_features, depth=128, use_depthwise=True)

    expected_feature_map_shapes = {
        'top_down_block2': (4, 8, 8, 128),
        'top_down_block3': (4, 4, 4, 128),
        'top_down_block4': (4, 2, 2, 128),
        'top_down_block5': (4, 1, 1, 128)
    }

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = {key: value.shape
                                for key, value in out_feature_maps.items()}
      self.assertDictEqual(out_feature_map_shapes, expected_feature_map_shapes)
//  [python]
def test_get_expected_feature_map_shapes_with_depthwise(self):
    image_features = [
        ('block2', tf.random_uniform([4, 8, 8, 256], dtype=tf.float32)),
        ('block3', tf.random_uniform([4, 4, 4, 256], dtype=tf.float32)),
        ('block4', tf.random_uniform([4, 2, 2, 256], dtype=tf.float32)),
        ('block5', tf.random_uniform([4, 1, 1, 256], dtype=tf.float32))
    ]
    feature_maps = feature_map_generators.fpn_top_down_feature_maps(
        image_features=image_features, depth=128, use_depthwise=True)

    expected_feature_map_shapes = {
        'top_down_block2': (4, 8, 8, 128),
        'top_down_block3': (4, 4, 4, 128),
        'top_down_block4': (4, 2, 2, 128),
        'top_down_block5': (4, 1, 1, 128)
    }

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      out_feature_maps = sess.run(feature_maps)
      out_feature_map_shapes = {key: value.shape
                                for key, value in out_feature_maps.items()}
      self.assertDictEqual(out_feature_map_shapes, expected_feature_map_shapes)
//  [python]
def infer(self, source_obj, embedding_ids, model_dir, save_dir):
        source_provider = InjectDataProvider(source_obj)

        if isinstance(embedding_ids, int) or len(embedding_ids) == 1:
            embedding_id = embedding_ids if isinstance(embedding_ids, int) else embedding_ids[0]
            source_iter = source_provider.get_single_embedding_iter(self.batch_size, embedding_id)
        else:
            source_iter = source_provider.get_random_embedding_iter(self.batch_size, embedding_ids)

        tf.initialize_all_variables().run()
        saver = tf.train.Saver(var_list=self.retrieve_generator_vars())
        self.restore_model(saver, model_dir)

        def save_imgs(imgs, count):
            p = os.path.join(save_dir, "inferred_%04d.png" % count)
            save_concat_images(imgs, img_path=p)
            print("generated images saved at %s" % p)

        count = 0
        batch_buffer = list()
        for labels, source_imgs in source_iter:
            fake_imgs = self.generate_fake_samples(source_imgs, labels)[0]
            merged_fake_images = merge(scale_back(fake_imgs), [self.batch_size, 1])
            batch_buffer.append(merged_fake_images)
            if len(batch_buffer) == 10:
                save_imgs(batch_buffer, count)
                batch_buffer = list()
            count += 1
        if batch_buffer:
            # last batch
            save_imgs(batch_buffer, count)
//  [python]
def infer(self, source_obj, embedding_ids, model_dir, save_dir):
        source_provider = InjectDataProvider(source_obj)

        if isinstance(embedding_ids, int) or len(embedding_ids) == 1:
            embedding_id = embedding_ids if isinstance(embedding_ids, int) else embedding_ids[0]
            source_iter = source_provider.get_single_embedding_iter(self.batch_size, embedding_id)
        else:
            source_iter = source_provider.get_random_embedding_iter(self.batch_size, embedding_ids)

        tf.global_variables_initializer().run()
        saver = tf.train.Saver(var_list=self.retrieve_generator_vars())
        self.restore_model(saver, model_dir)

        def save_imgs(imgs, count):
            p = os.path.join(save_dir, "inferred_%04d.png" % count)
            save_concat_images(imgs, img_path=p)
            print("generated images saved at %s" % p)

        count = 0
        batch_buffer = list()
        for labels, source_imgs in source_iter:
            fake_imgs = self.generate_fake_samples(source_imgs, labels)[0]
            merged_fake_images = merge(scale_back(fake_imgs), [self.batch_size, 1])
            batch_buffer.append(merged_fake_images)
            if len(batch_buffer) == 10:
                save_imgs(batch_buffer, count)
                batch_buffer = list()
            count += 1
        if batch_buffer:
            # last batch
            save_imgs(batch_buffer, count)
//  [python]
def test_extract_proposal_features_returns_expected_size(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    preprocessed_inputs = tf.random_uniform(
        [1, 299, 299, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 19, 19, 4320])
//  [python]
def test_extract_proposal_features_returns_expected_size(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    preprocessed_inputs = tf.random_uniform(
        [1, 299, 299, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 19, 19, 4320])
//  [python]
def test_extract_proposal_features_input_size_224(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    preprocessed_inputs = tf.random_uniform(
        [1, 224, 224, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 14, 14, 4320])
//  [python]
def test_extract_proposal_features_input_size_224(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    preprocessed_inputs = tf.random_uniform(
        [1, 224, 224, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 14, 14, 4320])
//  [python]
def test_extract_proposal_features_input_size_112(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    preprocessed_inputs = tf.random_uniform(
        [1, 112, 112, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 7, 7, 4320])
//  [python]
def test_extract_proposal_features_input_size_112(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    preprocessed_inputs = tf.random_uniform(
        [1, 112, 112, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 7, 7, 4320])
//  [python]
def test_extract_box_classifier_features_returns_expected_size(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    proposal_feature_maps = tf.random_uniform(
        [2, 17, 17, 1088], maxval=255, dtype=tf.float32)
    proposal_classifier_features = (
        feature_extractor.extract_box_classifier_features(
            proposal_feature_maps, scope='TestScope'))
    features_shape = tf.shape(proposal_classifier_features)

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [2, 9, 9, 4320])
//  [python]
def test_extract_box_classifier_features_returns_expected_size(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=16)
    proposal_feature_maps = tf.random_uniform(
        [2, 17, 17, 1088], maxval=255, dtype=tf.float32)
    proposal_classifier_features = (
        feature_extractor.extract_box_classifier_features(
            proposal_feature_maps, scope='TestScope'))
    features_shape = tf.shape(proposal_classifier_features)

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [2, 9, 9, 4320])
//  [python]
def main():
  env = gym.make('CartPole-v0')
  D = env.observation_space.shape[0]
  K = env.action_space.n
  pmodel = PolicyModel(D, K, [])
  vmodel = ValueModel(D, [10])
  init = tf.initialize_all_variables()
  session = tf.InteractiveSession()
  session.run(init)
  pmodel.set_session(session)
  vmodel.set_session(session)
  gamma = 0.99

  if 'monitor' in sys.argv:
    filename = os.path.basename(__file__).split('.')[0]
    monitor_dir = './' + filename + '_' + str(datetime.now())
    env = wrappers.Monitor(env, monitor_dir)

  N = 1000
  totalrewards = np.empty(N)
  costs = np.empty(N)
  for n in range(N):
    totalreward = play_one_mc(env, pmodel, vmodel, gamma)
    totalrewards[n] = totalreward
    if n % 100 == 0:
      print("episode:", n, "total reward:", totalreward, "avg reward (last 100):", totalrewards[max(0, n-100):(n+1)].mean())

  print("avg reward for last 100 episodes:", totalrewards[-100:].mean())
  print("total steps:", totalrewards.sum())

  plt.plot(totalrewards)
  plt.title("Rewards")
  plt.show()

  plot_running_avg(totalrewards)
//  [python]
def main():
  env = gym.make('CartPole-v0')
  D = env.observation_space.shape[0]
  K = env.action_space.n
  pmodel = PolicyModel(D, K, [])
  vmodel = ValueModel(D, [10])
  init = tf.global_variables_initializer()
  session = tf.InteractiveSession()
  session.run(init)
  pmodel.set_session(session)
  vmodel.set_session(session)
  gamma = 0.99

  if 'monitor' in sys.argv:
    filename = os.path.basename(__file__).split('.')[0]
    monitor_dir = './' + filename + '_' + str(datetime.now())
    env = wrappers.Monitor(env, monitor_dir)

  N = 1000
  totalrewards = np.empty(N)
  costs = np.empty(N)
  for n in range(N):
    totalreward = play_one_mc(env, pmodel, vmodel, gamma)
    totalrewards[n] = totalreward
    if n % 100 == 0:
      print("episode:", n, "total reward:", totalreward, "avg reward (last 100):", totalrewards[max(0, n-100):(n+1)].mean())

  print("avg reward for last 100 episodes:", totalrewards[-100:].mean())
  print("total steps:", totalrewards.sum())

  plt.plot(totalrewards)
  plt.title("Rewards")
  plt.show()

  plot_running_avg(totalrewards)
//  [python]
def _save_checkpoint_from_mock_model(self,
                                       checkpoint_path,
                                       use_moving_averages,
                                       quantize=False,
                                       num_channels=3):
    g = tf.Graph()
    with g.as_default():
      mock_model = FakeModel()
      inputs = tf.placeholder(tf.float32, shape=[1, 10, 10, num_channels])
      mock_model.predict(inputs, true_image_shapes=None)
      if use_moving_averages:
        tf.train.ExponentialMovingAverage(0.0).apply()
      tf.train.get_or_create_global_step()
      if quantize:
        graph_rewriter_config = graph_rewriter_pb2.GraphRewriter()
        graph_rewriter_config.quantization.delay = 500000
        graph_rewriter_fn = graph_rewriter_builder.build(
            graph_rewriter_config, is_training=False)
        graph_rewriter_fn()

      saver = tf.train.Saver()
      init = tf.initialize_all_variables()
      with self.test_session() as sess:
        sess.run(init)
        saver.save(sess, checkpoint_path)
//  [python]
def _save_checkpoint_from_mock_model(self,
                                       checkpoint_path,
                                       use_moving_averages,
                                       quantize=False,
                                       num_channels=3):
    g = tf.Graph()
    with g.as_default():
      mock_model = FakeModel()
      inputs = tf.placeholder(tf.float32, shape=[1, 10, 10, num_channels])
      mock_model.predict(inputs, true_image_shapes=None)
      if use_moving_averages:
        tf.train.ExponentialMovingAverage(0.0).apply()
      tf.train.get_or_create_global_step()
      if quantize:
        graph_rewriter_config = graph_rewriter_pb2.GraphRewriter()
        graph_rewriter_config.quantization.delay = 500000
        graph_rewriter_fn = graph_rewriter_builder.build(
            graph_rewriter_config, is_training=False)
        graph_rewriter_fn()

      saver = tf.train.Saver()
      init = tf.global_variables_initializer()
      with self.test_session() as sess:
        sess.run(init)
        saver.save(sess, checkpoint_path)
//  [python]
def initialize_session(self):
        """Defines self.sess and initialize the variables"""
        print("Initializing tf session")
        #config = tf.ConfigProto()
        #config.gpu_options.allow_growth = True
        #self.sess = tf.Session(config=config)
        self.sess = tf.Session()
        #from tensorflow.python import debug as tf_debug
        #self.sess = tf_debug.LocalCLIDebugWrapperSession(tf.Session())
        self.sess.run(tf.global_variables_initializer())
        self.ed_saver = tf.train.Saver(var_list=self.checkpoint_variables(), max_to_keep=self.args.checkpoints_num)
        self.el_saver = tf.train.Saver(var_list=self.checkpoint_variables(), max_to_keep=self.args.checkpoints_num)
//  [python]
def load_model(self, model_name, model_folder):
        self.balancing_representation = self.build_balancing_representation()
        self.treatment_prob_predictions = self.build_treatment_assignments_one_hot(self.balancing_representation)
        self.predictions = self.build_outcomes(self.balancing_representation)

        tf_device = 'gpu'
        if tf_device == "cpu":
            tf_config = tf.ConfigProto(log_device_placement=False, device_count={'GPU': 0})
        else:
            tf_config = tf.ConfigProto(log_device_placement=False, device_count={'GPU': 1})
            tf_config.gpu_options.allow_growth = True

        self.sess = tf.Session(config=tf_config)
        self.sess.run(tf.initialize_all_variables())
        checkpoint_name = model_name + "_final"
        self.load_network(self.sess, model_folder, checkpoint_name)
//  [python]
def load_model(self, model_name, model_folder):
        self.balancing_representation = self.build_balancing_representation()
        self.treatment_prob_predictions = self.build_treatment_assignments_one_hot(self.balancing_representation)
        self.predictions = self.build_outcomes(self.balancing_representation)

        tf_device = 'gpu'
        if tf_device == "cpu":
            tf_config = tf.ConfigProto(log_device_placement=False, device_count={'GPU': 0})
        else:
            tf_config = tf.ConfigProto(log_device_placement=False, device_count={'GPU': 1})
            tf_config.gpu_options.allow_growth = True

        self.sess = tf.Session(config=tf_config)
        self.sess.run(tf.global_variables_initializer())
        checkpoint_name = model_name + "_final"
        self.load_network(self.sess, model_folder, checkpoint_name)
//  [python]
def setup_meta_ops(self):
		cfg = dict({
			'allow_soft_placement': False,
			'log_device_placement': False
		})

		utility = min(self.FLAGS.gpu, 1.)
		if utility > 0.0:
			self.say('GPU mode with {} usage'.format(utility))
			cfg['gpu_options'] = tf.GPUOptions(
				per_process_gpu_memory_fraction = utility)
			cfg['allow_soft_placement'] = True
		else: 
			self.say('Running entirely on CPU')
			cfg['device_count'] = {'GPU': 0}

		if self.FLAGS.train: self.build_train_op()
		
		if self.FLAGS.summary:
			self.summary_op = tf.summary.merge_all()
			self.writer = tf.summary.FileWriter(self.FLAGS.summary + 'train')
		
		self.sess = tf.Session(config = tf.ConfigProto(**cfg))
		self.sess.run(tf.initialize_all_variables())

		if not self.ntrain: return
		self.saver = tf.train.Saver(tf.global_variables(), 
			max_to_keep = self.FLAGS.keep)
		if self.FLAGS.load != 0: self.load_from_ckpt()
		
		if self.FLAGS.summary:
			self.writer.add_graph(self.sess.graph)