jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def setup_meta_ops(self):
		cfg = dict({
			'allow_soft_placement': False,
			'log_device_placement': False
		})

		utility = min(self.FLAGS.gpu, 1.)
		if utility > 0.0:
			self.say('GPU mode with {} usage'.format(utility))
			cfg['gpu_options'] = tf.GPUOptions(
				per_process_gpu_memory_fraction = utility)
			cfg['allow_soft_placement'] = True
		else: 
			self.say('Running entirely on CPU')
			cfg['device_count'] = {'GPU': 0}

		if self.FLAGS.train: self.build_train_op()
		
		if self.FLAGS.summary:
			self.summary_op = tf.summary.merge_all()
			self.writer = tf.summary.FileWriter(self.FLAGS.summary + 'train')
		
		self.sess = tf.Session(config = tf.ConfigProto(**cfg))
		self.sess.run(tf.global_variables_initializer())

		if not self.ntrain: return
		self.saver = tf.train.Saver(tf.global_variables(), 
			max_to_keep = self.FLAGS.keep)
		if self.FLAGS.load != 0: self.load_from_ckpt()
		
		if self.FLAGS.summary:
			self.writer.add_graph(self.sess.graph)
//  [python]
def testFullyConvolutionalUnknownHeightWidth(self):
    batch = 2
    height, width = 65, 65
    global_pool = False
    inputs = create_test_input(batch, None, None, 3)
    with slim.arg_scope(xception.xception_arg_scope()):
      output, _ = self._xception_small(
          inputs,
          None,
          global_pool=global_pool)
    self.assertListEqual(output.get_shape().as_list(),
                         [batch, None, None, 16])
    images = create_test_input(batch, height, width, 3)
    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(output, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch, 3, 3, 16))
//  [python]
def testFullyConvolutionalUnknownHeightWidth(self):
    batch = 2
    height, width = 65, 65
    global_pool = False
    inputs = create_test_input(batch, None, None, 3)
    with slim.arg_scope(xception.xception_arg_scope()):
      output, _ = self._xception_small(
          inputs,
          None,
          global_pool=global_pool)
    self.assertListEqual(output.get_shape().as_list(),
                         [batch, None, None, 16])
    images = create_test_input(batch, height, width, 3)
    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(output, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch, 3, 3, 16))
//  [python]
def test_base_records(self):
        # WRITING
        for i in range(self.number_of_records):
            record_name = "base_type_{}.tfrecord".format(i)
            with BaseTypesRecordWriter(record_name, DIR_TFRECORDS) as writer:
                for j in range(self.samples_per_record):
                    writer.write_test(i * self.number_of_records + j)

        # READING
        # Important: read_batch MUST be called before start_queue_runners,
        # otherwise the internal shuffle queue gets created but its
        # threads won't start

        reader = BaseTypesRecordReader("base_type_*.tfrecord", DIR_TFRECORDS)
        read_one_example = reader.read_operation
        read_batched_examples = reader.read_batch(50)

        with tf.Session() as sess:
            sess.run(
                [tf.global_variables_initializer(), tf.local_variables_initializer()]
            )

            # Coordinate the queue of tfrecord files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            # Reading examples sequentially one by one
            for j in range(50):
                fetches = sess.run(read_one_example)
                print("Read:", fetches)

            # Reading a batch of examples
            results = sess.run(read_batched_examples)
            for i in range(len(results["string"])):
                print(
                    results["string"][i],
                    results["int"][i],
                    results["float"][i],
                    sep="\t",
                )

            # Finish off the queue coordinator.
            coord.request_stop()
            coord.join(threads)
//  [python]
def testUnknownBatchSize(self):
    batch = 2
    height, width = 65, 65
    global_pool = True
    num_classes = 10
    inputs = create_test_input(None, height, width, 3)
    with slim.arg_scope(xception.xception_arg_scope()):
      logits, _ = self._xception_small(
          inputs,
          num_classes,
          global_pool=global_pool,
          scope='xception')
    self.assertTrue(logits.op.name.startswith('xception/logits'))
    self.assertListEqual(logits.get_shape().as_list(),
                         [None, 1, 1, num_classes])
    images = create_test_input(batch, height, width, 3)
    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch, 1, 1, num_classes))
//  [python]
def testUnknownBatchSize(self):
    batch = 2
    height, width = 65, 65
    global_pool = True
    num_classes = 10
    inputs = create_test_input(None, height, width, 3)
    with slim.arg_scope(xception.xception_arg_scope()):
      logits, _ = self._xception_small(
          inputs,
          num_classes,
          global_pool=global_pool,
          scope='xception')
    self.assertTrue(logits.op.name.startswith('xception/logits'))
    self.assertListEqual(logits.get_shape().as_list(),
                         [None, 1, 1, num_classes])
    images = create_test_input(batch, height, width, 3)
    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(logits, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch, 1, 1, num_classes))
//  [python]
def testAtrousFullyConvolutionalUnknownHeightWidth(self):
    batch = 2
    height, width = 65, 65
    global_pool = False
    output_stride = 8
    inputs = create_test_input(batch, None, None, 3)
    with slim.arg_scope(xception.xception_arg_scope()):
      output, _ = self._xception_small(
          inputs,
          None,
          global_pool=global_pool,
          output_stride=output_stride)
    self.assertListEqual(output.get_shape().as_list(),
                         [batch, None, None, 16])
    images = create_test_input(batch, height, width, 3)
    with self.test_session() as sess:
      sess.run(tf.initialize_all_variables())
      output = sess.run(output, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch, 9, 9, 16))
//  [python]
def testAtrousFullyConvolutionalUnknownHeightWidth(self):
    batch = 2
    height, width = 65, 65
    global_pool = False
    output_stride = 8
    inputs = create_test_input(batch, None, None, 3)
    with slim.arg_scope(xception.xception_arg_scope()):
      output, _ = self._xception_small(
          inputs,
          None,
          global_pool=global_pool,
          output_stride=output_stride)
    self.assertListEqual(output.get_shape().as_list(),
                         [batch, None, None, 16])
    images = create_test_input(batch, height, width, 3)
    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      output = sess.run(output, {inputs: images.eval()})
      self.assertEquals(output.shape, (batch, 9, 9, 16))
//  [python]
def testAtrousFullyConvolutionalValues(self):
    """Verify dense feature extraction with atrous convolution."""
    nominal_stride = 32
    for output_stride in [4, 8, 16, 32, None]:
      with slim.arg_scope(xception.xception_arg_scope()):
        with tf.Graph().as_default():
          with self.test_session() as sess:
            tf.set_random_seed(0)
            inputs = create_test_input(2, 96, 97, 3)
            # Dense feature extraction followed by subsampling.
            output, _ = self._xception_small(
                inputs,
                None,
                is_training=False,
                global_pool=False,
                output_stride=output_stride)
            if output_stride is None:
              factor = 1
            else:
              factor = nominal_stride // output_stride
            output = resnet_utils.subsample(output, factor)
            # Make the two networks use the same weights.
            tf.get_variable_scope().reuse_variables()
            # Feature extraction at the nominal network rate.
            expected, _ = self._xception_small(
                inputs,
                None,
                is_training=False,
                global_pool=False)
            sess.run(tf.initialize_all_variables())
            self.assertAllClose(output.eval(), expected.eval(),
                                atol=1e-5, rtol=1e-5)
//  [python]
def testAtrousFullyConvolutionalValues(self):
    """Verify dense feature extraction with atrous convolution."""
    nominal_stride = 32
    for output_stride in [4, 8, 16, 32, None]:
      with slim.arg_scope(xception.xception_arg_scope()):
        with tf.Graph().as_default():
          with self.test_session() as sess:
            tf.set_random_seed(0)
            inputs = create_test_input(2, 96, 97, 3)
            # Dense feature extraction followed by subsampling.
            output, _ = self._xception_small(
                inputs,
                None,
                is_training=False,
                global_pool=False,
                output_stride=output_stride)
            if output_stride is None:
              factor = 1
            else:
              factor = nominal_stride // output_stride
            output = resnet_utils.subsample(output, factor)
            # Make the two networks use the same weights.
            tf.get_variable_scope().reuse_variables()
            # Feature extraction at the nominal network rate.
            expected, _ = self._xception_small(
                inputs,
                None,
                is_training=False,
                global_pool=False)
            sess.run(tf.global_variables_initializer())
            self.assertAllClose(output.eval(), expected.eval(),
                                atol=1e-5, rtol=1e-5)
//  [python]
def test_simple(params):
    """Test function."""

    left  = tf.placeholder(tf.float32, [2, args.input_height, args.input_width, 3])
    model = MonodepthModel(params, "test", left, None)

    input_image = scipy.misc.imread(args.image_path, mode="RGB")
    original_height, original_width, num_channels = input_image.shape
    input_image = scipy.misc.imresize(input_image, [args.input_height, args.input_width], interp='lanczos')
    input_image = input_image.astype(np.float32) / 255
    input_images = np.stack((input_image, np.fliplr(input_image)), 0)

    # SESSION
    config = tf.ConfigProto(allow_soft_placement=True)
    sess = tf.Session(config=config)

    # SAVER
    train_saver = tf.train.Saver()

    # INIT
    sess.run(tf.initialize_all_variables())
    sess.run(tf.local_variables_initializer())
    coordinator = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coordinator)

    # RESTORE
    restore_path = args.checkpoint_path.split(".")[0]
    train_saver.restore(sess, restore_path)

    disp = sess.run(model.disp_left_est[0], feed_dict={left: input_images})
    disp_pp = post_process_disparity(disp.squeeze()).astype(np.float32)

    output_directory = os.path.dirname(args.image_path)
    output_name = os.path.splitext(os.path.basename(args.image_path))[0]

    np.save(os.path.join(output_directory, "{}_disp.npy".format(output_name)), disp_pp)
    disp_to_img = scipy.misc.imresize(disp_pp.squeeze(), [original_height, original_width])
    plt.imsave(os.path.join(output_directory, "{}_disp.png".format(output_name)), disp_to_img, cmap='plasma')

    print('done!')
//  [python]
def test_simple(params):
    """Test function."""

    left  = tf.placeholder(tf.float32, [2, args.input_height, args.input_width, 3])
    model = MonodepthModel(params, "test", left, None)

    input_image = scipy.misc.imread(args.image_path, mode="RGB")
    original_height, original_width, num_channels = input_image.shape
    input_image = scipy.misc.imresize(input_image, [args.input_height, args.input_width], interp='lanczos')
    input_image = input_image.astype(np.float32) / 255
    input_images = np.stack((input_image, np.fliplr(input_image)), 0)

    # SESSION
    config = tf.ConfigProto(allow_soft_placement=True)
    sess = tf.Session(config=config)

    # SAVER
    train_saver = tf.train.Saver()

    # INIT
    sess.run(tf.global_variables_initializer())
    sess.run(tf.local_variables_initializer())
    coordinator = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coordinator)

    # RESTORE
    restore_path = args.checkpoint_path.split(".")[0]
    train_saver.restore(sess, restore_path)

    disp = sess.run(model.disp_left_est[0], feed_dict={left: input_images})
    disp_pp = post_process_disparity(disp.squeeze()).astype(np.float32)

    output_directory = os.path.dirname(args.image_path)
    output_name = os.path.splitext(os.path.basename(args.image_path))[0]

    np.save(os.path.join(output_directory, "{}_disp.npy".format(output_name)), disp_pp)
    disp_to_img = scipy.misc.imresize(disp_pp.squeeze(), [original_height, original_width])
    plt.imsave(os.path.join(output_directory, "{}_disp.png".format(output_name)), disp_to_img, cmap='plasma')

    print('done!')
//  [python]
W = tf.Variable(tf.random_normal(shape=[d, m, k]))
b = tf.Variable(tf.random_normal(shape = [m, k]))
dot_z = tf.tensordot(x, W, axes=1) + b
z = tf.reduce_max(dot_z, axis=2)

with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    print(sess.run([x,dot_z,z]))
//  [python]
W = tf.Variable(tf.random_normal(shape=[d, m, k]))
b = tf.Variable(tf.random_normal(shape = [m, k]))
dot_z = tf.tensordot(x, W, axes=1) + b
z = tf.reduce_max(dot_z, axis=2)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    print(sess.run([x,dot_z,z]))
//  [python]
def execute(self, train_data, test_data):

        self.prepare_data(train_data, test_data)
        init = tf.initialize_all_variables()
        self.sess.run(init)
        for epoch in range(self.epochs):
            if self.verbose:
                print("Epoch: %04d;" % (epoch))
            self.train(train_data)
            if (epoch) % self.T == 0 and epoch >= 5:
                print("Epoch: %04d; " % (epoch), end='')
                self.test(test_data)
//  [python]
def execute(self, train_data, test_data):

        self.prepare_data(train_data, test_data)
        init = tf.global_variables_initializer()
        self.sess.run(init)
        for epoch in range(self.epochs):
            if self.verbose:
                print("Epoch: %04d;" % (epoch))
            self.train(train_data)
            if (epoch) % self.T == 0 and epoch >= 5:
                print("Epoch: %04d; " % (epoch), end='')
                self.test(test_data)
//  [python]
def test():
    #below is a function test; if you use this for text classifiction, you need to tranform sentence to indices of vocabulary first. then feed data to the graph.
    num_classes=10
    learning_rate=0.01
    batch_size=8
    decay_steps=1000
    decay_rate=0.9
    sequence_length=5
    vocab_size=10000
    embed_size=100
    is_training=True
    dropout_keep_prob=1#0.5
    textRNN=TextRNN(num_classes, learning_rate, batch_size, decay_steps, decay_rate,sequence_length,vocab_size,embed_size,is_training)
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        for i in range(100):
            input_x=np.zeros((batch_size,sequence_length)) #[None, self.sequence_length]
            input_y=input_y=np.array([1,0,1,1,1,2,1,1]) #np.zeros((batch_size),dtype=np.int32) #[None, self.sequence_length]
            loss,acc,predict,_=sess.run([textRNN.loss_val,textRNN.accuracy,textRNN.predictions,textRNN.train_op],feed_dict={textRNN.input_x:input_x,textRNN.input_y:input_y,textRNN.dropout_keep_prob:dropout_keep_prob})
            print("loss:",loss,"acc:",acc,"label:",input_y,"prediction:",predict)
//  [python]
def test():
    #below is a function test; if you use this for text classifiction, you need to tranform sentence to indices of vocabulary first. then feed data to the graph.
    num_classes=10
    learning_rate=0.01
    batch_size=8
    decay_steps=1000
    decay_rate=0.9
    sequence_length=5
    vocab_size=10000
    embed_size=100
    is_training=True
    dropout_keep_prob=1#0.5
    textRNN=TextRNN(num_classes, learning_rate, batch_size, decay_steps, decay_rate,sequence_length,vocab_size,embed_size,is_training)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(100):
            input_x=np.zeros((batch_size,sequence_length)) #[None, self.sequence_length]
            input_y=input_y=np.array([1,0,1,1,1,2,1,1]) #np.zeros((batch_size),dtype=np.int32) #[None, self.sequence_length]
            loss,acc,predict,_=sess.run([textRNN.loss_val,textRNN.accuracy,textRNN.predictions,textRNN.train_op],feed_dict={textRNN.input_x:input_x,textRNN.input_y:input_y,textRNN.dropout_keep_prob:dropout_keep_prob})
            print("loss:",loss,"acc:",acc,"label:",input_y,"prediction:",predict)
//  [python]
def restore(self, session, latest_checkpoint=False):
        if latest_checkpoint:
            checkpoint_path = tf.train.latest_checkpoint(self.config["log_dir"])
        else:
            checkpoint_path = os.path.join(self.config["log_dir"], "model.max.ckpt")

        print("Restoring from {}".format(checkpoint_path))
        session.run(tf.initialize_all_variables())
        self.saver.restore(session, checkpoint_path)
//  [python]
def restore(self, session, latest_checkpoint=False):
        if latest_checkpoint:
            checkpoint_path = tf.train.latest_checkpoint(self.config["log_dir"])
        else:
            checkpoint_path = os.path.join(self.config["log_dir"], "model.max.ckpt")

        print("Restoring from {}".format(checkpoint_path))
        session.run(tf.global_variables_initializer())
        self.saver.restore(session, checkpoint_path)
//  [python]
def test_fixed_size_record(self):
        # WRITING
        with FixedSizeTypesRecordWriter("fixed_size.tfrecord", DIR_TFRECORDS) as writer:
            writer.write_test()
            writer.write_test()

        # READING
        # Important: read_batch MUST be called before start_queue_runners,
        # otherwise the internal shuffle queue gets created but its
        # threads won't start
        reader = FixedSizeTypesRecordReader("fixed_size.tfrecord", DIR_TFRECORDS)
        read_one_example = reader.read_operation
        read_batched_examples = reader.read_batch(4)

        with tf.Session() as sess:
            sess.run(
                [tf.initialize_all_variables(), tf.local_variables_initializer()]
            )

            # Coordinate the queue of tfrecord files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            # Reading examples sequentially one by one
            for j in range(3):
                fetches = sess.run(read_one_example)
                print("Read:", fetches)

            # Reading a batch of examples
            results = sess.run(read_batched_examples)
            print(results)

            # Finish off the queue coordinator.
            coord.request_stop()
            coord.join(threads)
//  [python]
def test_fixed_size_record(self):
        # WRITING
        with FixedSizeTypesRecordWriter("fixed_size.tfrecord", DIR_TFRECORDS) as writer:
            writer.write_test()
            writer.write_test()

        # READING
        # Important: read_batch MUST be called before start_queue_runners,
        # otherwise the internal shuffle queue gets created but its
        # threads won't start
        reader = FixedSizeTypesRecordReader("fixed_size.tfrecord", DIR_TFRECORDS)
        read_one_example = reader.read_operation
        read_batched_examples = reader.read_batch(4)

        with tf.Session() as sess:
            sess.run(
                [tf.global_variables_initializer(), tf.local_variables_initializer()]
            )

            # Coordinate the queue of tfrecord files.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(coord=coord)

            # Reading examples sequentially one by one
            for j in range(3):
                fetches = sess.run(read_one_example)
                print("Read:", fetches)

            # Reading a batch of examples
            results = sess.run(read_batched_examples)
            print(results)

            # Finish off the queue coordinator.
            coord.request_stop()
            coord.join(threads)
//  [python]
def restore(self):  # name
		# if not session: session= tf.Session()
		self.session = tf.Session()
		checkpoint = tf.train.get_checkpoint_state(checkpoint_dir)
		if checkpoint and checkpoint.model_checkpoint_path:
			print("Restoring old model from meta graph")
			loader = tf.train.import_meta_graph(checkpoint.model_checkpoint_path + ".meta")
		else:
			print("No model from meta graph, nothing to restore")
			return self
		self.session.run(tf.initialize_all_variables())
		print("loading checkpoint %s" % checkpoint.model_checkpoint_path)
		try:
			loader.restore(self.session, tf.train.latest_checkpoint(checkpoint_dir))
		except:
			pass
		# loader.restore(self.session , checkpoint) #Unable to get element from the feed as bytes!  HUH??
		self.input = self.x = tf.get_collection('inputs')[0]
		self.target = self.y = tf.get_collection('targets')[0]
		self.output = self.last_layer = tf.get_collection('outputs')[0]
		self.dropout_keep_prob = self.session.graph.get_tensor_by_name("state/dropout_keep_prob:0")  # :0 WTF!?!?!
		self.train_phase = self.session.graph.get_tensor_by_name(name='state/train_phase:0')
		return self
//  [python]
def restore(self):  # name
		# if not session: session= tf.Session()
		self.session = tf.Session()
		checkpoint = tf.train.get_checkpoint_state(checkpoint_dir)
		if checkpoint and checkpoint.model_checkpoint_path:
			print("Restoring old model from meta graph")
			loader = tf.train.import_meta_graph(checkpoint.model_checkpoint_path + ".meta")
		else:
			print("No model from meta graph, nothing to restore")
			return self
		self.session.run(tf.global_variables_initializer())
		print("loading checkpoint %s" % checkpoint.model_checkpoint_path)
		try:
			loader.restore(self.session, tf.train.latest_checkpoint(checkpoint_dir))
		except:
			pass
		# loader.restore(self.session , checkpoint) #Unable to get element from the feed as bytes!  HUH??
		self.input = self.x = tf.get_collection('inputs')[0]
		self.target = self.y = tf.get_collection('targets')[0]
		self.output = self.last_layer = tf.get_collection('outputs')[0]
		self.dropout_keep_prob = self.session.graph.get_tensor_by_name("state/dropout_keep_prob:0")  # :0 WTF!?!?!
		self.train_phase = self.session.graph.get_tensor_by_name(name='state/train_phase:0')
		return self
//  [python]
def test_extract_proposal_features_stride_eight(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=8)
    preprocessed_inputs = tf.random_uniform(
        [1, 224, 224, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.initialize_all_variables()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 28, 28, 1088])
//  [python]
def test_extract_proposal_features_stride_eight(self):
    feature_extractor = self._build_feature_extractor(
        first_stage_features_stride=8)
    preprocessed_inputs = tf.random_uniform(
        [1, 224, 224, 3], maxval=255, dtype=tf.float32)
    rpn_feature_map, _ = feature_extractor.extract_proposal_features(
        preprocessed_inputs, scope='TestScope')
    features_shape = tf.shape(rpn_feature_map)

    init_op = tf.global_variables_initializer()
    with self.test_session() as sess:
      sess.run(init_op)
      features_shape_out = sess.run(features_shape)
      self.assertAllEqual(features_shape_out, [1, 28, 28, 1088])
//  [python]
def decode_tf_records_to_pandas(decoding_features_spec,
                                tf_records_path,
                                max_n_examples=None,
                                random_filter_keep_rate=1.0,
                                filter_fn=None):
  """Loads tf-records into a pandas dataframe.

  Args:
    decoding_features_spec: A dict mapping feature keys to FixedLenFeature
      values. Spec of the tf-records.
    tf_records_path: path to the file
    max_n_examples: Maximum number of examples to extract.
    random_filter_keep_rate: Probability for each line to be kept in training
      data. For each line, we generate a random number x and keep it if x <
      random_filter_keep_rate.
    filter_fn (optional): Function applied to an example. If it returns False,
      the example will be discarded.

  Returns:
    A pandas `DataFrame`.
  """

  if not max_n_examples:
    max_n_examples = float('inf')

  reader = tf.TFRecordReader()
  filenames = tf.train.match_filenames_once(tf_records_path)
  filename_queue = tf.train.string_input_producer(filenames,
                                                  num_epochs=1)

  _, serialized_example = reader.read(filename_queue)
  read_data = tf.parse_single_example(
      serialized=serialized_example, features=decoding_features_spec)

  sess = tf.InteractiveSession()
  sess.run(tf.initialize_all_variables())
  sess.run(tf.local_variables_initializer())
  sess.run(tf.tables_initializer())
  tf.train.start_queue_runners(sess)

  d = []
  new_line = sess.run(read_data)
  count = 0
  while new_line:
    if filter_fn:
      keep_line = filter_fn(new_line)
    else:
      keep_line = True
    keep_line = keep_line and (random.random() < random_filter_keep_rate)

    if keep_line:
      d.append(new_line)
      count += 1
      if count >= max_n_examples:
        break
      if not (count % 100000):
        logging.info('Loaded {} lines.'.format(count))

    try:
      new_line = sess.run(read_data)
    except tf.errors.OutOfRangeError:
      logging.info('End of file.')
      break

  res = pd.DataFrame(d)
  return res
//  [python]
def decode_tf_records_to_pandas(decoding_features_spec,
                                tf_records_path,
                                max_n_examples=None,
                                random_filter_keep_rate=1.0,
                                filter_fn=None):
  """Loads tf-records into a pandas dataframe.

  Args:
    decoding_features_spec: A dict mapping feature keys to FixedLenFeature
      values. Spec of the tf-records.
    tf_records_path: path to the file
    max_n_examples: Maximum number of examples to extract.
    random_filter_keep_rate: Probability for each line to be kept in training
      data. For each line, we generate a random number x and keep it if x <
      random_filter_keep_rate.
    filter_fn (optional): Function applied to an example. If it returns False,
      the example will be discarded.

  Returns:
    A pandas `DataFrame`.
  """

  if not max_n_examples:
    max_n_examples = float('inf')

  reader = tf.TFRecordReader()
  filenames = tf.train.match_filenames_once(tf_records_path)
  filename_queue = tf.train.string_input_producer(filenames,
                                                  num_epochs=1)

  _, serialized_example = reader.read(filename_queue)
  read_data = tf.parse_single_example(
      serialized=serialized_example, features=decoding_features_spec)

  sess = tf.InteractiveSession()
  sess.run(tf.global_variables_initializer())
  sess.run(tf.local_variables_initializer())
  sess.run(tf.tables_initializer())
  tf.train.start_queue_runners(sess)

  d = []
  new_line = sess.run(read_data)
  count = 0
  while new_line:
    if filter_fn:
      keep_line = filter_fn(new_line)
    else:
      keep_line = True
    keep_line = keep_line and (random.random() < random_filter_keep_rate)

    if keep_line:
      d.append(new_line)
      count += 1
      if count >= max_n_examples:
        break
      if not (count % 100000):
        logging.info('Loaded {} lines.'.format(count))

    try:
      new_line = sess.run(read_data)
    except tf.errors.OutOfRangeError:
      logging.info('End of file.')
      break

  res = pd.DataFrame(d)
  return res
//  [python]
def gif_making(npz_path):
    assert npz_path != ''

    min_window_size = 32
    raster_size = 128

    split_idx = npz_path.rfind('/')
    if split_idx == -1:
        file_base = './'
        file_name = npz_path[:-4]
    else:
        file_base = npz_path[:npz_path.rfind('/')]
        file_name = npz_path[npz_path.rfind('/') + 1: -4]

    gif_base = os.path.join(file_base, file_name)
    os.makedirs(gif_base, exist_ok=True)

    # differentiable pasting graph
    paste_v3_func = DiffPastingV3(raster_size)

    tfconfig = tf.ConfigProto()
    tfconfig.gpu_options.allow_growth = True
    sess = tf.InteractiveSession(config=tfconfig)
    sess.run(tf.global_variables_initializer())

    data = np.load(npz_path, encoding='latin1', allow_pickle=True)
    strokes_data = data['strokes_data']
    init_cursors = data['init_cursors']
    image_size = data['image_size']
    round_length = data['round_length']
    init_width = data['init_width']

    if round_length.ndim == 0:
        round_lengths = [round_length]
    else:
        round_lengths = round_length

    # print('round_lengths', round_lengths)

    make_gif(sess, paste_v3_func,
             strokes_data, init_cursors, image_size, round_lengths, init_width,
             gif_base,
             min_window_size=min_window_size, raster_size=raster_size)
//  [python]
def bootstrap():
    """Initialize a tf.Estimator run with random initial weights."""
    # a bit hacky - forge an initial checkpoint with the name that subsequent
    # Estimator runs will expect to find.
    #
    # Estimator will do this automatically when you call train(), but calling
    # train() requires data, and I didn't feel like creating training data in
    # order to run the full train pipeline for 1 step.
    maybe_set_seed()
    initial_checkpoint_name = 'model.ckpt-1'
    save_file = os.path.join(FLAGS.work_dir, initial_checkpoint_name)
    sess = tf.Session(graph=tf.Graph())
    with sess.graph.as_default():
        features, labels = get_inference_input()
        model_fn(features, labels, tf.estimator.ModeKeys.PREDICT,
                 params=FLAGS.flag_values_dict())
        sess.run(tf.initialize_all_variables())
        tf.train.Saver().save(sess, save_file)
//  [python]
def bootstrap():
    """Initialize a tf.Estimator run with random initial weights."""
    # a bit hacky - forge an initial checkpoint with the name that subsequent
    # Estimator runs will expect to find.
    #
    # Estimator will do this automatically when you call train(), but calling
    # train() requires data, and I didn't feel like creating training data in
    # order to run the full train pipeline for 1 step.
    maybe_set_seed()
    initial_checkpoint_name = 'model.ckpt-1'
    save_file = os.path.join(FLAGS.work_dir, initial_checkpoint_name)
    sess = tf.Session(graph=tf.Graph())
    with sess.graph.as_default():
        features, labels = get_inference_input()
        model_fn(features, labels, tf.estimator.ModeKeys.PREDICT,
                 params=FLAGS.flag_values_dict())
        sess.run(tf.global_variables_initializer())
        tf.train.Saver().save(sess, save_file)
//  [python]
def test_neural_network():
    """Test the NeuralNetwork class init."""
    relu = tf.nn.relu

    with tf.Session() as sess:
        nn = NeuralNetwork(layers=[2, 3, 1],
                           nonlinearities=[relu, relu, None])

        # x = tf.placeholder()
        res = nn(np.random.rand(4, 2))
        sess.run(tf.initialize_all_variables())
        res, lipschitz = sess.run([res, nn.lipschitz()])

    assert lipschitz > 0.
//  [python]
def test_neural_network():
    """Test the NeuralNetwork class init."""
    relu = tf.nn.relu

    with tf.Session() as sess:
        nn = NeuralNetwork(layers=[2, 3, 1],
                           nonlinearities=[relu, relu, None])

        # x = tf.placeholder()
        res = nn(np.random.rand(4, 2))
        sess.run(tf.global_variables_initializer())
        res, lipschitz = sess.run([res, nn.lipschitz()])

    assert lipschitz > 0.
//  [python]
def setup(self):
        """Create testing environment."""
        with tf.Session(graph=tf.Graph()) as sess:
            npoints = 3

            discretization = GridWorld([[0, 1], [0, 1]], npoints)
            parameters = np.sum(discretization.all_points ** 2,
                                axis=1, keepdims=True)
            trinp = _Triangulation(discretization, vertex_values=parameters)

            tri = Triangulation(discretization, vertex_values=parameters)

            test_points = np.array([[-10, -10],
                                    [0.2, 0.7],
                                    [0, 0],
                                    [0, 1],
                                    [1, 1],
                                    [-0.2, 0.5],
                                    [0.43, 0.21]])

            sess.run(tf.initialize_all_variables())
            yield sess, tri, trinp, test_points
//  [python]
def setup(self):
        """Create testing environment."""
        with tf.Session(graph=tf.Graph()) as sess:
            npoints = 3

            discretization = GridWorld([[0, 1], [0, 1]], npoints)
            parameters = np.sum(discretization.all_points ** 2,
                                axis=1, keepdims=True)
            trinp = _Triangulation(discretization, vertex_values=parameters)

            tri = Triangulation(discretization, vertex_values=parameters)

            test_points = np.array([[-10, -10],
                                    [0.2, 0.7],
                                    [0, 0],
                                    [0, 1],
                                    [1, 1],
                                    [-0.2, 0.5],
                                    [0.43, 0.21]])

            sess.run(tf.global_variables_initializer())
            yield sess, tri, trinp, test_points
//  [python]
def buildModel(self):
        print ('training...')
        y = tf.reduce_sum(tf.multiply(self.u_embedding, self.v_embedding), 1) \
            - tf.reduce_sum(tf.multiply(self.u_embedding, self.neg_item_embedding), 1)
        reg_loss = self.regU * (tf.nn.l2_loss(self.u_embedding) + tf.nn.l2_loss(self.v_embedding) +
                                                                    tf.nn.l2_loss(self.neg_item_embedding))
        for i in range(self.n_layer):
            reg_loss+= self.regU*tf.nn.l2_loss(self.weights['layer_%d' %(i+1)])
        loss = -tf.reduce_sum(tf.log(tf.sigmoid(y))) + reg_loss
        opt = tf.train.AdamOptimizer(self.lRate)
        train = opt.minimize(loss)
        init = tf.initialize_all_variables()
        self.sess.run(init)
        for iteration in range(self.maxIter):
            for n, batch in enumerate(self.next_batch_pairwise()):
                user_idx, i_idx, j_idx = batch
                _, l = self.sess.run([train, loss],
                                feed_dict={self.u_idx: user_idx, self.neg_idx: j_idx, self.v_idx: i_idx, self.isTraining:1})
                print ('training:', iteration + 1, 'batch', n, 'loss:', l)
//  [python]
def buildModel(self):
        print ('training...')
        y = tf.reduce_sum(tf.multiply(self.u_embedding, self.v_embedding), 1) \
            - tf.reduce_sum(tf.multiply(self.u_embedding, self.neg_item_embedding), 1)
        reg_loss = self.regU * (tf.nn.l2_loss(self.u_embedding) + tf.nn.l2_loss(self.v_embedding) +
                                                                    tf.nn.l2_loss(self.neg_item_embedding))
        for i in range(self.n_layer):
            reg_loss+= self.regU*tf.nn.l2_loss(self.weights['layer_%d' %(i+1)])
        loss = -tf.reduce_sum(tf.log(tf.sigmoid(y))) + reg_loss
        opt = tf.train.AdamOptimizer(self.lRate)
        train = opt.minimize(loss)
        init = tf.global_variables_initializer()
        self.sess.run(init)
        for iteration in range(self.maxIter):
            for n, batch in enumerate(self.next_batch_pairwise()):
                user_idx, i_idx, j_idx = batch
                _, l = self.sess.run([train, loss],
                                feed_dict={self.u_idx: user_idx, self.neg_idx: j_idx, self.v_idx: i_idx, self.isTraining:1})
                print ('training:', iteration + 1, 'batch', n, 'loss:', l)
//  [python]
def train():
    
    my_global_step = tf.Variable(0, name='global_step', trainable=False)
    
    
    data_dir = '/home/kevin/tensorflow/CIFAR10/data/cifar-10-batches-bin/'
    log_dir = '/home/kevin/tensorflow/CIFAR10/logs234/'
    
    images, labels = cifar10_input.read_cifar10(data_dir=data_dir,
                                                is_train=True,
                                                batch_size= BATCH_SIZE,
                                                shuffle=True)
    logits = inference(images)
    
    loss = losses(logits, labels)
    
    
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    train_op = optimizer.minimize(loss, global_step= my_global_step)
    
    saver = tf.train.Saver(tf.global_variables())
    summary_op = tf.summary.merge_all()
    
    
    
    init = tf.initialize_all_variables()
    sess = tf.Session()
    sess.run(init)
    
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    
    summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
    
    try:
        for step in np.arange(MAX_STEP):
            if coord.should_stop():
                    break
            _, loss_value = sess.run([train_op, loss])
               
            if step % 50 == 0:                 
                print ('Step: %d, loss: %.4f' % (step, loss_value))
                
            if step % 100 == 0:
                summary_str = sess.run(summary_op)
                summary_writer.add_summary(summary_str, step)                
    
            if step % 2000 == 0 or (step + 1) == MAX_STEP:
                checkpoint_path = os.path.join(log_dir, 'model.ckpt')
                saver.save(sess, checkpoint_path, global_step=step)
                
    except tf.errors.OutOfRangeError:
        print('Done training -- epoch limit reached')
    finally:
        coord.request_stop()
        
    coord.join(threads)
    sess.close()
//  [python]
def train():
    
    my_global_step = tf.Variable(0, name='global_step', trainable=False)
    
    
    data_dir = '/home/kevin/tensorflow/CIFAR10/data/cifar-10-batches-bin/'
    log_dir = '/home/kevin/tensorflow/CIFAR10/logs234/'
    
    images, labels = cifar10_input.read_cifar10(data_dir=data_dir,
                                                is_train=True,
                                                batch_size= BATCH_SIZE,
                                                shuffle=True)
    logits = inference(images)
    
    loss = losses(logits, labels)
    
    
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    train_op = optimizer.minimize(loss, global_step= my_global_step)
    
    saver = tf.train.Saver(tf.global_variables())
    summary_op = tf.summary.merge_all()
    
    
    
    init = tf.global_variables_initializer()
    sess = tf.Session()
    sess.run(init)
    
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    
    summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
    
    try:
        for step in np.arange(MAX_STEP):
            if coord.should_stop():
                    break
            _, loss_value = sess.run([train_op, loss])
               
            if step % 50 == 0:                 
                print ('Step: %d, loss: %.4f' % (step, loss_value))
                
            if step % 100 == 0:
                summary_str = sess.run(summary_op)
                summary_writer.add_summary(summary_str, step)                
    
            if step % 2000 == 0 or (step + 1) == MAX_STEP:
                checkpoint_path = os.path.join(log_dir, 'model.ckpt')
                saver.save(sess, checkpoint_path, global_step=step)
                
    except tf.errors.OutOfRangeError:
        print('Done training -- epoch limit reached')
    finally:
        coord.request_stop()
        
    coord.join(threads)
    sess.close()
//  [python]
def train():
  lr_g = tf.placeholder(tf.float32, shape=[])
  lr_d = tf.placeholder(tf.float32, shape=[])
  is_training_g = tf.placeholder(tf.bool, shape=[])
  is_training_d = tf.placeholder(tf.bool, shape=[])
  is_training_v = tf.placeholder(tf.bool, shape=[])
  is_training_a = tf.placeholder(tf.bool, shape=[])
  phg = {'lr_g': lr_g,
       'is_training_g': is_training_g}
  phd = {'lr_d': lr_d,
       'is_training_d': is_training_d}
  phv = {'is_training_v': is_training_v}
  pha = {'is_training_a': is_training_a}
  with tf.Session() as sess:
    G_u, G_loss, G_update = G_graph(sess, phg)
    Ga_u, Ga_loss, Ga_acc, Ga_update = G_adv_graph(sess, phg)
    D_loss, D_update  = D_graph(sess, phd)
    tr_images, re_images, fa_images = V_graph(sess, phv)
    adv_loss, adv_acc, adv_update, adv_pred = Adv_graph(sess, pha)
    saver = tf.train.Saver(Genc.vars+Gdec.vars)
    init = tf.initialize_all_variables()
    sess.run(init)
    # saver.restore(sess,LOG_PATH+'/weight.ckpt')

    #####################  pre-train Generator & Discriminator  #####################
    g_dict = {phg['lr_g']: 1e-4,
          phg['is_training_g']: True}
    d_dict = {phd['lr_d']: 1e-4,
          phd['is_training_d']: True}
    for epoch in range(N_EPOCH):
      for i in range(train_iters):
        if i % 6 !=0:
          d_loss, _ = sess.run([D_loss, D_update], feed_dict=d_dict)
        else:
          g_loss, _ = sess.run([G_loss, G_update], feed_dict=g_dict)

        if i % 100 == 0 and i != 0:
          log_string('====pre_epoch_%d====iter_%d: g_loss=%.4f, d_loss=%.4f' % (epoch, i, g_loss, d_loss))
    saver.save(sess,LOG_PATH+'/pretrain.ckpt')

    #####################  train and fix adversarial classifier  #####################
    a_dict = {pha['is_training_a']: True}
    for epoch in range(N_ADV):
      for i in range(train_iters):
        a_loss, a_acc, _ = sess.run([adv_loss, adv_acc, adv_update], feed_dict=a_dict)
        if i % 100 == 0 and i != 0:
          log_string('====adv_epoch_%d====iter_%d: a_loss=%.4f, a_acc=%.4f' % (epoch, i, a_loss, a_acc))

    #####################  fine-tune Generator & Discriminator  #####################
    g_dict = {phg['lr_g']: 5e-6,
          phg['is_training_g']: True}
    d_dict = {phd['lr_d']: 5e-6,
          phd['is_training_d']: True}
    v_dict = {phv['is_training_v']: True}
    for epoch in range(N_EPOCH):
      for i in range(train_iters):
        if i % 6 !=0:
          d_loss, _ = sess.run([D_loss, D_update], feed_dict=d_dict)
        else:
          g_loss, a_acc, _ = sess.run([Ga_loss, Ga_acc, Ga_update], feed_dict=g_dict)

        if i % 100 == 0 and i != 0:
          log_string('====epoch_%d====iter_%d: g_loss=%.4f, d_loss=%.4f, a_acc=%.4f' % (epoch, i, g_loss, d_loss, a_acc))
      a,b,c = sess.run([tr_images, re_images, fa_images], feed_dict = v_dict)
      image_list = [a,b,c]
      sample = np.transpose(image_list, (1, 2, 0, 3, 4))
      sample = np.reshape(sample, (-1, sample.shape[2] * sample.shape[3], sample.shape[4]))
      sample = util.to_uint8(sample)
      io.imsave('%s/Epoch-%d.jpg' % (LOG_PATH, epoch), sample, quality=95)
      saver.save(sess,LOG_PATH+'/weight.ckpt')
//  [python]
def train():
  lr_g = tf.placeholder(tf.float32, shape=[])
  lr_d = tf.placeholder(tf.float32, shape=[])
  is_training_g = tf.placeholder(tf.bool, shape=[])
  is_training_d = tf.placeholder(tf.bool, shape=[])
  is_training_v = tf.placeholder(tf.bool, shape=[])
  is_training_a = tf.placeholder(tf.bool, shape=[])
  phg = {'lr_g': lr_g,
       'is_training_g': is_training_g}
  phd = {'lr_d': lr_d,
       'is_training_d': is_training_d}
  phv = {'is_training_v': is_training_v}
  pha = {'is_training_a': is_training_a}
  with tf.Session() as sess:
    G_u, G_loss, G_update = G_graph(sess, phg)
    Ga_u, Ga_loss, Ga_acc, Ga_update = G_adv_graph(sess, phg)
    D_loss, D_update  = D_graph(sess, phd)
    tr_images, re_images, fa_images = V_graph(sess, phv)
    adv_loss, adv_acc, adv_update, adv_pred = Adv_graph(sess, pha)
    saver = tf.train.Saver(Genc.vars+Gdec.vars)
    init = tf.global_variables_initializer()
    sess.run(init)
    # saver.restore(sess,LOG_PATH+'/weight.ckpt')

    #####################  pre-train Generator & Discriminator  #####################
    g_dict = {phg['lr_g']: 1e-4,
          phg['is_training_g']: True}
    d_dict = {phd['lr_d']: 1e-4,
          phd['is_training_d']: True}
    for epoch in range(N_EPOCH):
      for i in range(train_iters):
        if i % 6 !=0:
          d_loss, _ = sess.run([D_loss, D_update], feed_dict=d_dict)
        else:
          g_loss, _ = sess.run([G_loss, G_update], feed_dict=g_dict)

        if i % 100 == 0 and i != 0:
          log_string('====pre_epoch_%d====iter_%d: g_loss=%.4f, d_loss=%.4f' % (epoch, i, g_loss, d_loss))
    saver.save(sess,LOG_PATH+'/pretrain.ckpt')

    #####################  train and fix adversarial classifier  #####################
    a_dict = {pha['is_training_a']: True}
    for epoch in range(N_ADV):
      for i in range(train_iters):
        a_loss, a_acc, _ = sess.run([adv_loss, adv_acc, adv_update], feed_dict=a_dict)
        if i % 100 == 0 and i != 0:
          log_string('====adv_epoch_%d====iter_%d: a_loss=%.4f, a_acc=%.4f' % (epoch, i, a_loss, a_acc))

    #####################  fine-tune Generator & Discriminator  #####################
    g_dict = {phg['lr_g']: 5e-6,
          phg['is_training_g']: True}
    d_dict = {phd['lr_d']: 5e-6,
          phd['is_training_d']: True}
    v_dict = {phv['is_training_v']: True}
    for epoch in range(N_EPOCH):
      for i in range(train_iters):
        if i % 6 !=0:
          d_loss, _ = sess.run([D_loss, D_update], feed_dict=d_dict)
        else:
          g_loss, a_acc, _ = sess.run([Ga_loss, Ga_acc, Ga_update], feed_dict=g_dict)

        if i % 100 == 0 and i != 0:
          log_string('====epoch_%d====iter_%d: g_loss=%.4f, d_loss=%.4f, a_acc=%.4f' % (epoch, i, g_loss, d_loss, a_acc))
      a,b,c = sess.run([tr_images, re_images, fa_images], feed_dict = v_dict)
      image_list = [a,b,c]
      sample = np.transpose(image_list, (1, 2, 0, 3, 4))
      sample = np.reshape(sample, (-1, sample.shape[2] * sample.shape[3], sample.shape[4]))
      sample = util.to_uint8(sample)
      io.imsave('%s/Epoch-%d.jpg' % (LOG_PATH, epoch), sample, quality=95)
      saver.save(sess,LOG_PATH+'/weight.ckpt')
//  [python]
def reward_USE_similarity(conversations):
    """Allocates reward for coherence between user input and bot response in
    Universal Sentence Encoder embedding space"""
    if 'universal_encoder' not in globals():
        print('Loading Universal Sentence Encoder')
        global universal_encoder, sess, sents, embed_op
        use_path = os.path.join(ROOT_DIR, "UniversalSentenceEncoder")

        with tf.device('/cpu:0'):
            universal_encoder = hub.Module(use_path)
            sents = tf.placeholder(tf.string, shape=None, name="input_sents")
            embed_op = universal_encoder(sents)

        sess = tf.Session()
        sess.run([tf.global_variables_initializer(), tf.tables_initializer()])

    num_convs = len(conversations)
    episode_len = (len(conversations[0]) - 1) // 2
    # Flattened responses
    bot_responses = [resp for conv in conversations for resp in conv[1::2]]
    user_responses = [resp for conv in conversations for resp in conv[2::2]]

    user_embed = sess.run(embed_op, feed_dict={sents: user_responses})
    bot_embed = sess.run(embed_op, feed_dict={sents: bot_responses})
    similarity = cosine_similarity(user_embed, bot_embed)
    rewards = similarity.reshape(num_convs, episode_len)

    return rewards