Recurrent Attention CNN の実装に関するメモ

##ポイント
* Recurrent Attention CNN を実装し、具体的な数値で確認。
* 今後、パフォーマンス検証を実施。

##レファレンス
[1. Look Closer to See Better: Recurrent Attention Convolutional Neural Network
for Fine-grained Image Recognition
](http://openaccess.thecvf.com/content_cvpr_2017/papers/Fu_Look_Closer_to_CVPR_2017_paper.pdf)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/9e818be3-3032-5474-f105-763890ec7f9d.png)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/c086048b-9fce-d49d-d506-55f81f4dca9a.png)

　　　　　　　　　　　　　（参照論文より引用）

##データ
* Distorted MNIST。[Distorted MNIST 作成に関するメモ](https://qiita.com/takeshikondo/items/5c4f38e18a2b4513205a)

##サンプルコード
```python
class RecurrentAttention():
  def __init__(self):
    pass
  
  def weight_variable(self, name, shape):
    initializer = tf.truncated_normal_initializer(mean = 0.0, stddev = 0.01, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)
  
  def bias_variable(self, name, shape):
    initializer = tf.constant_initializer(value = 0.0, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)
  
  def cnn_feature_learning(self, x, filter_size, n_filters_1, n_filters_2):
    w_1 = self.weight_variable('w_1', [filter_size, filter_size, 1, n_filters_1])
    b_1 = self.bias_variable('b_1', [n_filters_1])

    conv_1 = tf.nn.relu(tf.nn.conv2d(x, w_1, strides = [1, 1, 1, 1], padding = 'SAME') + b_1)
    conv_1 = tf.nn.max_pool(conv_1, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = 'SAME')
      
    w_2 = self.weight_variable('w_2', [filter_size, filter_size, n_filters_1, n_filters_2])
    b_2 = self.bias_variable('b_2', [n_filters_2])

    conv_2 = tf.nn.relu(tf.nn.conv2d(conv_1, w_2, strides = [1, 1, 1, 1], padding = 'SAME') + b_2)
    
    return tf.nn.max_pool(conv_2, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = 'SAME')

  def two_fc_layers(self, x, hight, width, n_filters, n_fc_units_1, n_fc_units_2, keep_prob):
    x = tf.reshape(x, [-1, hight * width * n_filters])
    
    w_fc_1 = self.weight_variable('w_fc_1', [hight * width * n_filters, n_fc_units_1])
    b_fc_1 = self.bias_variable('b_fc_1', [n_fc_units_1])
    
    fc_1 = tf.nn.relu(tf.matmul(x, w_fc_1) + b_fc_1)
    fc_1_dropout = tf.nn.dropout(fc_1, keep_prob)
      
    w_fc_2 = self.weight_variable('w_fc_2', [n_fc_units_1, n_fc_units_2])
    b_fc_2 = self.bias_variable('b_fc_2', [n_fc_units_2])
    
    return tf.matmul(fc_1_dropout, w_fc_2) + b_fc_2 
  
  def attention_proposal(self, x, hight, width, n_filters, t_l, original_hight):
    x = tf.reshape(x, [-1, hight * width * n_filters])
    
    w = self.weight_variable('w', [hight * width * n_filters, 2])
    b = self.bias_variable('b', [2])
    
    t = tf.nn.sigmoid(tf.matmul(x, w) + b)
    
    return tf.cast(tf.floor(t * (original_hight - t_l)), tf.int32)
  
  # to be done
  def attention_mask_generation(self):
    pass
  
  def localization_net(self, x, hight, width, n_filters):
    w_l = self.weight_variable('w_l', [hight * width * n_filters, 2])
    b_l = self.bias_variable('b_l', [2])

    x = tf.reshape(x, [-1, hight * width * n_filters])
    theta = tf.nn.tanh(tf.matmul(x, w_l) + b_l)
    #theta = tf.matmul(x, w_l) + b_l
  
    return theta
        
  def grid_generator(self, batch_size, hight, width, theta):
    x_t = -1 + tf.range(width) / width * 2
    x_t = tf.reshape(tf.concat([x_t] * hight, axis = 0), [1, hight * width])
    x_t = tf.expand_dims(x_t, axis = 0)
    x_t = tf.tile(x_t, [batch_size, 1, 1])
    x_t = tf.cast(x_t, dtype = tf.float32)
    
    y_t = -1 + tf.range(hight) / hight * 2
    y_t = tf.tile(tf.reshape(y_t, [hight, 1]), [1, width])
    y_t = tf.reshape(y_t, [1, hight * width])
    y_t = tf.expand_dims(y_t, axis = 0)
    y_t = tf.tile(y_t, [batch_size, 1, 1])
    y_t = tf.cast(y_t, dtype = tf.float32)

    ones_t = tf.ones(shape = [batch_size, 1, hight * width], dtype = tf.float32)
    grids_t = tf.concat([x_t, y_t, ones_t], axis = 1)
    
    theta = tf.expand_dims(theta, axis = 2)
    
    tmp = np.array([[0.5, 0.0], [0.0, 0.5]])
    tmp = tf.convert_to_tensor(tmp, dtype = tf.float32)
    tmp = tf.expand_dims(tmp, axis = 0)
    tmp = tf.tile(tmp, [batch_size, 1, 1])
    
    A = tf.concat([tmp, theta], axis = 2)

    grids_s = tf.matmul(A, grids_t)

    return grids_s
  
  def sampler(self, u, grids, batch_size, hight, width):
    img_in = tf.reshape(u, [-1, hight, width])
    
    x = (1.0 + grids[:, 0]) * width / 2.0
    y = (1.0 + grids[:, 1]) * hight / 2.0

    x0 = tf.cast(tf.floor(x), tf.int32) 
    x1 = x0 + 1
    y0 = tf.cast(tf.floor(y), tf.int32) 
    y1 = y0 + 1

    w_ul = (tf.cast(x1, tf.float32) - x) * (tf.cast(y1, tf.float32) - y)
    w_ll = (tf.cast(x1, tf.float32) - x) * (y - tf.cast(y0, tf.float32))
    w_ur = (x - tf.cast(x0, tf.float32)) * (tf.cast(y1, tf.float32) - y)
    w_lr = (x - tf.cast(x0, tf.float32)) * (y - tf.cast(y0, tf.float32))

    x0_img = tf.maximum(0, tf.minimum(x0, width - 1))
    x1_img = tf.maximum(0, tf.minimum(x1, width - 1))
    y0_img = tf.maximum(0, tf.minimum(y0, hight - 1))
    y1_img = tf.maximum(0, tf.minimum(y1, hight - 1))
    
    x0_img = tf.expand_dims(x0_img, axis = 2)
    x1_img = tf.expand_dims(x1_img, axis = 2)
    y0_img = tf.expand_dims(y0_img, axis = 2)
    y1_img = tf.expand_dims(y1_img, axis = 2)

    idx_ul = tf.concat([y0_img, x0_img], axis = 2)
    idx_ll = tf.concat([y1_img, x0_img], axis = 2)
    idx_ur = tf.concat([y0_img, x1_img], axis = 2)
    idx_lr = tf.concat([y1_img, x1_img], axis = 2)

    idx_ul = tf.reshape(idx_ul, [batch_size * hight * width, 2])
    idx_ll = tf.reshape(idx_ll, [batch_size * hight * width, 2])
    idx_ur = tf.reshape(idx_ur, [batch_size * hight * width, 2])
    idx_lr = tf.reshape(idx_lr, [batch_size * hight * width, 2])
    
    idx_batch = np.reshape(np.arange(batch_size), [batch_size, 1])
    idx_batch = np.repeat(idx_batch, hight * width, axis = 0)
    idx_batch = tf.convert_to_tensor(idx_batch, dtype = tf.int32)

    idx_ul = tf.concat([idx_batch, idx_ul], axis = 1)
    idx_ll = tf.concat([idx_batch, idx_ll], axis = 1)
    idx_ur = tf.concat([idx_batch, idx_ur], axis = 1)
    idx_lr = tf.concat([idx_batch, idx_lr], axis = 1)
    
    img_ul = tf.gather_nd(img_in, idx_ul)
    img_ll = tf.gather_nd(img_in, idx_ll)
    img_ur = tf.gather_nd(img_in, idx_ur)
    img_lr = tf.gather_nd(img_in, idx_lr)
    
    img_ul = tf.reshape(img_ul, [batch_size, hight * width])
    img_ll = tf.reshape(img_ll, [batch_size, hight * width])
    img_ur = tf.reshape(img_ur, [batch_size, hight * width])
    img_lr = tf.reshape(img_lr, [batch_size, hight * width])
       
    v = w_ul * img_ul + w_ll * img_ll + w_ur * img_ur + w_lr * img_lr
    
    return tf.reshape(v, [-1, hight, width, 1])
      
  def classification(self, x):
    predictions = tf.nn.softmax(x, axis = 2)
    
    return tf.reduce_mean(predictions, axis = 0)      

  def inference(self, x, hight, width, filter_size, n_filters_1, n_filters_2, \
               n_fc_units_1, n_fc_units_2, keep_prob, batch_size):
    
    list_y = []
    
    zoomed = tf.reshape(x, [-1, hight, width, 1])
    for i in range(3):
      with tf.variable_scope('scale_{}'.format(i)):
        cnn_features = self.cnn_feature_learning(zoomed, filter_size, n_filters_1, n_filters_2)

        hight_2 = 7   # 28 / 2 / 2
        width_2 = 7   # 28 / 2 / 2

        # classification
        y = self.two_fc_layers(cnn_features, hight_2, width_2, n_filters_2, n_fc_units_1, n_fc_units_2, keep_prob)  
        y = tf.nn.softmax(y, axis = 1)
        list_y.append(y)

        # attention proposal
        theta = self.localization_net(cnn_features, hight_2, width_2, n_filters_2)
        grids = self.grid_generator(batch_size, hight, width, theta)
        v = self.sampler(zoomed, grids, batch_size, hight, width)
        
        zoomed = v

    return list_y      
  
  def loss_cls(self, y, t):
    cross_entropy = - tf.reduce_mean(tf.reduce_sum(t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), axis = 1))
    return cross_entropy
  
  def loss_rank(self, y_s, y_s_1, t, margin):
    p_s = t * y_s
    p_s_1 = t * y_s_1
    return tf.reduce_mean(tf.nn.relu(p_s - p_s_1 + margin))
  
  def accuracy(self, y, t):
    correct_preds = tf.equal(tf.argmax(y, axis = 1), tf.argmax(t, axis = 1))
    
    return tf.reduce_mean(tf.cast(correct_preds, tf.float32))
     
  def training(self, loss, learning_rate):
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
    train_step = optimizer.minimize(loss)
    return train_step
  
  def training_clipped(self, loss, learning_rate, clip_norm):
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)

    grads_and_vars = optimizer.compute_gradients(loss)
    clipped_grads_and_vars = [(tf.clip_by_norm(grad, clip_norm = clip_norm), \
                             var) for grad, var in grads_and_vars]
    train_step = optimizer.apply_gradients(clipped_grads_and_vars)
    
    return train_step

  def fit(self, images_train, labels_train, images_test, labels_test, \
          filter_size, n_filters_1, n_filters_2, n_fc_units, margin, \
          learning_rate, n_iter, batch_size, show_step, is_saving, model_path):
        
    tf.reset_default_graph()
    
    x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    t = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
    
    list_y = self.inference(x, 28, 28, filter_size, n_filters_1, n_filters_2, \
                           n_fc_units, 10, keep_prob, batch_size)
    
    
    #loss = self.loss_cls(list_y[0], t)
    loss = self.loss_cls(list_y[0], t) + self.loss_cls(list_y[1], t) + self.loss_cls(list_y[2], t)
    loss += self.loss_rank(list_y[0], list_y[1], t, margin) + self.loss_rank(list_y[1], list_y[2], t, margin) 
        
    # Without Gradient Clipping
    train_step = self.training(loss, learning_rate)
    # With Gradient Clipping
    #train_step = self.training_clipped(loss, learning_rate, 0.1)

    y = list_y[2]
    acc =  self.accuracy(y, t)
    
    y_1 = list_y[0]
    acc_1 =  self.accuracy(y_1, t)
    
    y_2 = list_y[1]
    acc_2 =  self.accuracy(y_2, t)
    
    y_3 = list_y[2]
    acc_3 =  self.accuracy(y_3, t)
    
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()
    
    with tf.Session() as sess:
      
      sess.run(init)
      
      history_loss_train = []
      history_acc_train = []
      history_loss_test = []
      history_acc_test = []
      
      for i in range(n_iter):
        # Train
        rand_index = np.random.choice(len(images_train), size = batch_size)
        x_batch = images_train[rand_index]
        y_batch = labels_train[rand_index]

        feed_dict = {x: x_batch, t: y_batch, keep_prob: 0.7}

        sess.run(train_step, feed_dict = feed_dict)

        temp_loss = sess.run(loss, feed_dict = feed_dict)
        temp_acc = sess.run(acc, feed_dict = feed_dict)

        temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
        temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)
        temp_acc_3 = sess.run(acc_3, feed_dict = feed_dict)

        history_loss_train.append(temp_loss)
        history_acc_train.append(temp_acc)

        if (i + 1) % show_step == 0:
          print ('--------------------')
          print ('Iteration: ' + str(i + 1) + '  Loss: ' + str(temp_loss) + \
                '  Accuracy 1: ' + str(temp_acc_1) + '  Accuracy 2: ' + str(temp_acc_2) + \
                '  Accuracy 3: ' + str(temp_acc_3))
                   
        # Test
        rand_index = np.random.choice(len(images_test), size = batch_size)
        x_batch = images_test[rand_index]
        y_batch = labels_test[rand_index]

        feed_dict = {x: x_batch, t: y_batch, keep_prob: 1.0}

        temp_loss = sess.run(loss, feed_dict = feed_dict)
        temp_acc = sess.run(acc, feed_dict = feed_dict)

        history_loss_test.append(temp_loss)
        history_acc_test.append(temp_acc)

      if is_saving:
        model_path = saver.save(sess, model_path)
        print ('done saving at ', model_path)

    fig = plt.figure(figsize = (10, 3))
    ax1 = fig.add_subplot(1, 2, 1)
    ax1.plot(range(n_iter), history_loss_train, 'b-', label = 'Train')
    ax1.plot(range(n_iter), history_loss_test, 'r--', label = 'Test')
    ax1.set_title('Loss')
    ax1.legend(loc = 'upper right')

    ax2 = fig.add_subplot(1, 2, 2)
    ax2.plot(range(n_iter), history_acc_train, 'b-', label = 'Train')
    ax2.plot(range(n_iter), history_acc_test, 'r--', label = 'Test')
    ax2.set_title('Accuracy')
    ax2.legend(loc = 'lower right')

    plt.show()

  def get_samples(self, images_train, labels_train, images_test, labels_test, \
          filter_size, n_filters_1, n_filters_2, n_fc_units, margin, \
          batch_size, model_path):
    
    x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    t = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
    
    samples = []
    
    hight = 28
    width = 28
    
    zoomed = tf.reshape(x, [-1, hight, width, 1])
    for i in range(3):
      with tf.variable_scope('scale_{}'.format(i), reuse = True):
        cnn_features = self.cnn_feature_learning(zoomed, filter_size, n_filters_1, n_filters_2)

        hight_2 = 7   # 28 / 2 / 2
        width_2 = 7   # 28 / 2 / 2
        
        samples.append(zoomed)

        # attention proposal
        theta = self.localization_net(cnn_features, hight_2, width_2, n_filters_2)
        grids = self.grid_generator(batch_size, hight, width, theta)
        v = self.sampler(zoomed, grids, batch_size, hight, width)
        
        zoomed = v

    saver = tf.train.Saver()
    
    with tf.Session() as sess:
      
      saver.restore(sess, model_path)
      
      rand_index = np.random.choice(len(images_train), size = batch_size)
      x_batch = images_train[rand_index]
      y_batch = labels_train[rand_index]

      feed_dict = {x: x_batch, t: y_batch, keep_prob: 1.0}

      samples = sess.run(samples, feed_dict = feed_dict)

    fig = plt.figure(figsize = (10, 3))
    ax1 = fig.add_subplot(1, 3, 1)
    ax1.imshow(np.reshape(samples[0][0], [28, 28]), cmap = 'gray')
    ax1.set_title('Scale 1')
    ax1.set_axis_off()

    ax2 = fig.add_subplot(1, 3, 2)
    ax2.imshow(np.reshape(samples[1][0], [28, 28]), cmap = 'gray')
    ax2.set_title('Scale 2')
    ax2.set_axis_off()

    ax3 = fig.add_subplot(1, 3, 3)
    ax3.imshow(np.reshape(samples[2][0], [28, 28]), cmap = 'gray')
    ax3.set_title('Scale 3')
    ax3.set_axis_off()

    plt.show()
```    

##アウトプット
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/53cd1fe1-457f-473b-159e-b645e5e0ff6b.png)
<br>
![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/575c97b2-a012-0900-f360-13c415d7beae.png)