Tri-training Domain Adaptation の実装に関するメモ

##Reference
[Asymmetric Tri-training for Unsupervised Domain Adaptation](https://arxiv.org/abs/1702.08400)

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/bf2e9a20-f906-2151-735d-4bda2aac29ab.png)

##Data
```python
images_train = mnist.train.images
labels_train = mnist.train.labels
images_test = mnist.test.images
labels_test = mnist.test.labels

import skimage.transform

# for train data
indices = np.random.choice(55000, 10000, replace = False)

images_train_0 = images_train[indices]
images_train_0_2d = np.reshape(images_train_0, (-1, 28, 28))
labels_train_0 = labels_train[indices]

#images_train_flip_2d = images_train_0_2d[:, :, ::-1]
#images_train_flip = np.reshape(images_train_0_2d[:, :, ::-1], (-1, 28*28))
#labels_train_flip = labels_train[indices]

#images_train_30_2d = []
#for i in range(len(images_train_0)):
#  images_train_30_2d.append(skimage.transform.rotate(images_train_0_2d[i], 30))
#images_train_30 = np.reshape(images_train_30_2d, (-1, 28*28))
#labels_train_30 = labels_train[indices] 

#images_train_60_2d = []
#for i in range(len(images_train_0)):
#  images_train_60_2d.append(skimage.transform.rotate(images_train_0_2d[i], 60))
#images_train_60 = np.reshape(images_train_60_2d, (-1, 28*28))
#labels_train_60 = labels_train[indices] 

#images_train_90_2d = []
#for i in range(len(images_train_0)):
#  images_train_90_2d.append(skimage.transform.rotate(images_train_0_2d[i], 90))
#images_train_90 = np.reshape(images_train_90_2d, (-1, 28*28))
#labels_train_90 = labels_train[indices] 

#images_train_180_2d = []
#for i in range(len(images_train_0)):
#  images_train_180_2d.append(skimage.transform.rotate(images_train_0_2d[i], 180))
#images_train_180 = np.reshape(images_train_180_2d, (-1, 28*28))
#labels_train_180 = labels_train[indices] 

import skimage.transform

# for test data
indices = np.random.choice(10000, 10000, replace = False)

images_test_0 = images_test[indices]
images_test_0_2d = np.reshape(images_test_0, (-1, 28, 28))
labels_test_0 = labels_test[indices]

#images_test_flip_2d = images_test_0_2d[:, :, ::-1]
#images_test_flip = np.reshape(images_test_flip_2d, (-1, 28*28))
#labels_test_flip = labels_test[indices]

#images_test_30_2d = []
#for i in range(len(images_test_0)):
#  images_test_30_2d.append(skimage.transform.rotate(images_test_0_2d[i], 30))
#images_test_30 = np.reshape(images_test_30_2d, (-1, 28*28))
#labels_test_30 = labels_test[indices] 

#images_test_60_2d = []
#for i in range(len(images_test_0)):
#  images_test_60_2d.append(skimage.transform.rotate(images_test_0_2d[i], 60))
#images_test_60 = np.reshape(images_test_60_2d, (-1, 28*28))
#labels_test_60 = labels_test[indices] 

#images_test_90_2d = []
#for i in range(len(images_test_0)):
#  images_test_90_2d.append(skimage.transform.rotate(images_test_0_2d[i], 90))
#images_test_90 = np.reshape(images_test_90_2d, (-1, 28*28))
#labels_test_90 = labels_test[indices] 

#images_test_180_2d = []
#for i in range(len(images_test_0)):
#  images_test_180_2d.append(skimage.transform.rotate(images_test_0_2d[i], 180))
#images_test_180 = np.reshape(images_test_180_2d, (-1, 28*28))
#labels_test_180 = labels_test[indices] 
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/af70eced-8e5b-66c0-1a40-b7c02ee4254d.png)


##Sample Code
```python
# Tri-training Domain Adaptation

class TDA():
  def __init__(self):
    pass

  def weight_variable(self, name, shape):
    initializer = tf.truncated_normal_initializer(mean = 0.0, stddev = 0.01, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)

  def bias_variable(self, name, shape):
    initializer = tf.constant_initializer(value = 0.0, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)
  
  def F_0(self, x, filter_size, n_filters_1, n_filters_2, n_units, keep_prob, reuse = False):
    x_reshaped = tf.reshape(x, [-1, 28, 28, 1])
    
    with tf.variable_scope('F_0', reuse = reuse):
      w_1 = self.weight_variable('w_1', [filter_size, filter_size, 1, n_filters_1])
      b_1 = self.bias_variable('b_1', [n_filters_1])

      # conv
      conv = tf.nn.conv2d(x_reshaped, w_1, strides = [1, 2, 2, 1], padding = 'SAME') + b_1
    
      # batch norm
      #batch_mean, batch_var = tf.nn.moments(conv, [0, 1, 2])
      #conv = (conv - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
    
      # relu
      conv = tf.nn.relu(conv)
    
      # max_pool
      #conv = tf.nn.max_pool(conv, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = 'SAME')
    
      w_2 = self.weight_variable('w_2', [filter_size, filter_size, n_filters_1, n_filters_2])
      b_2 = self.bias_variable('b_2', [n_filters_2])
      
      # conv
      conv = tf.nn.conv2d(conv, w_2, strides = [1, 2, 2, 1], padding = 'SAME') + b_2
    
      # batch norm
      batch_mean, batch_var = tf.nn.moments(conv, [0, 1, 2])
      conv = (conv - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
    
      # relu
      conv = tf.nn.relu(conv)
    
      # max_pool
      #conv = tf.nn.max_pool(conv, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = 'SAME')
    
      conv_flat = tf.reshape(conv, [-1, 7 * 7 * n_filters_2])
    
      w_3 = self.weight_variable('w_3', [7 * 7 * n_filters_2, n_units])
      b_3 = self.bias_variable('b_3', [n_units])
    
      fc = tf.nn.relu(tf.matmul(conv_flat, w_3) + b_3)
      
      # batch norm
      batch_mean, batch_var = tf.nn.moments(fc, [0])
    
      fc = (fc - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
    
      # dropout
      #fc = tf.nn.dropout(fc, keep_prob)
      
      feature = fc
      
    return feature

  def F_1_2(self, x_1, x_2, n_units_1, n_units_2, keep_prob, reuse = False):
    
    with tf.variable_scope('F_1_2', reuse = reuse):
      w_1_1 = self.weight_variable('w_1_1', [n_units_1, n_units_2])
      b_1_1 = self.bias_variable('b_1_1', [n_units_2])
      w_1_2 = self.weight_variable('w_1_2', [n_units_1, n_units_2])
      b_1_2 = self.bias_variable('b_1_2', [n_units_2])
    
      fc_1 = tf.matmul(x_1, w_1_1) + b_1_1
      fc_2 = tf.matmul(x_2, w_1_2) + b_1_2
      
      # Full alignment
      # batch norm
      batch_mean_1, batch_var_1 = tf.nn.moments(fc_1, [0])
      fc_1 = (fc_1 - batch_mean_1) / (tf.sqrt(batch_var_1) + 1e-10)
      
      batch_mean_2, batch_var_2 = tf.nn.moments(fc_2, [0])
      fc_2 = (fc_2 - batch_mean_2) / (tf.sqrt(batch_var_2) + 1e-10)

      # Zero alignment
      # batch norm
      #batch_mean_1, batch_var_1 = tf.nn.moments(fc_1, [0])
      #batch_mean_2, batch_var_2 = tf.nn.moments(fc_2, [0])

      #fc_1 = 0.5 * (fc_1 - batch_mean_1) / (tf.sqrt(batch_var_1) + 1e-10) +\
      #      0.5 * (fc_1 - batch_mean_2) / (tf.sqrt(batch_var_2) + 1e-10)
      #fc_2 = 0.5 * (fc_2 - batch_mean_1) / (tf.sqrt(batch_var_1) + 1e-10) +\
      #      0.5 * (fc_2 - batch_mean_2) / (tf.sqrt(batch_var_2) + 1e-10)

      # relu
      fc_1 = tf.nn.relu(fc_1)
      fc_2 = tf.nn.relu(fc_2)
      
      # dropout
      #fc_1 = tf.nn.dropout(fc_1, keep_prob)
      #fc_2 = tf.nn.dropout(fc_2, keep_prob)

      w_2_1 = self.weight_variable('w_2_1', [n_units_2, 10])
      b_2_1 = self.bias_variable('b_2_1', [10])
      w_2_2 = self.weight_variable('w_2_2', [n_units_2, 10])
      b_2_2 = self.bias_variable('b_2_2', [10])

      fc_1 = tf.matmul(fc_1, w_2_1) + b_2_1
      fc_2 = tf.matmul(fc_2, w_2_2) + b_2_2
      
      logits_1 = fc_1
      logits_2 = fc_2
    
    return logits_1, logits_2, w_1_1, w_1_2
  
  def F_t(self, x, n_units_1, n_units_2, keep_prob, reuse = False):
    
    with tf.variable_scope('F_t', reuse = reuse):
      w_1 = self.weight_variable('w_1', [n_units_1, n_units_2])
      b_1 = self.bias_variable('b_1', [n_units_2])
    
      fc = tf.matmul(x, w_1) + b_1
      
      # batch norm
      batch_mean, batch_var = tf.nn.moments(fc, [0])
      fc = (fc - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
      
      # relu
      fc = tf.nn.relu(fc)
    
      # dropout
      #fc = tf.nn.dropout(fc, keep_prob)
    
      w_2 = self.weight_variable('w_2', [n_units_2, 10])
      b_2 = self.bias_variable('b_2', [10])

      fc = tf.matmul(fc, w_2) + b_2
      logits = fc
    
    return logits

  def loss_cross_entropy(self, y, t):
    cross_entropy = - tf.reduce_mean(tf.reduce_sum(t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), axis = 1))
    return cross_entropy

  def loss_cross_entropy_mask(self, y, t, mask):
    cross_entropy = - tf.reduce_mean(tf.reduce_sum(t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), axis = 1) * mask)
    return cross_entropy

  def loss_entropy(self, p):
    entropy = - tf.reduce_mean(tf.reduce_sum(p * tf.log(tf.clip_by_value(p, 1e-10, 1.0)), axis = 1))
    return entropy

  def loss_mutual_information(self, p):
    p_ave = tf.reduce_mean(p, axis = 0)
    h_y = -tf.reduce_sum(p_ave * tf.log(p_ave + 1e-16))
    h_y_x = - tf.reduce_mean(tf.reduce_sum(p * tf.log(tf.clip_by_value(p, 1e-10, 1.0)), axis = 1))
    mutual_info = h_y - h_y_x
    return -mutual_info

  def accuracy(self, y, t):
    correct_preds = tf.equal(tf.argmax(y, axis = 1), tf.argmax(t, axis = 1))
    accuracy = tf.reduce_mean(tf.cast(correct_preds, tf.float32))
    return accuracy

  def training(self, loss, learning_rate, var_list):
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
    #optimizer = tf.train.GradientDescentOptimizer(learning_rate = learning_rate)
    train_step = optimizer.minimize(loss, var_list = var_list)
    return train_step

  def training_clipped(self, loss, learning_rate, clip_norm, var_list):
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)

    grads_and_vars = optimizer.compute_gradients(loss, var_list = var_list)
    clipped_grads_and_vars = [(tf.clip_by_norm(grad, clip_norm = clip_norm), \
                             var) for grad, var in grads_and_vars]
    train_step = optimizer.apply_gradients(clipped_grads_and_vars)

    return train_step
  
  def fit_1_2(self, images_train_1, labels_train_1, images_test_1, labels_test_1, \
                images_train_2, labels_train_2, images_test_2, labels_test_2, \
                filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, lam, \
                learning_rate, n_iter, batch_size, show_step, is_saving, model_path):

    tf.reset_default_graph()

    x_1 = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    x_2 = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    y_1 = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    y_2 = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)

    feat_1 = self.F_0(x_1, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = False)
    feat_2 = self.F_0(x_2, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = True)
    
    logits_1, logits_2, w_1_1, w_1_2 = self.F_1_2(feat_1, feat_2, n_units_0, n_units_1, keep_prob, reuse = False)
    
    probs_1 = tf.nn.softmax(logits_1)
    loss_1 = self.loss_cross_entropy(probs_1, y_1)
    
    probs_2 = tf.nn.softmax(logits_2)
    loss_2 = self.loss_cross_entropy(probs_2, y_2)  # labeled data
    #loss_2 = self.loss_entropy(probs_t)  # unlabeled data
    
    loss_w = tf.reduce_sum(tf.abs(tf.matmul(tf.transpose(w_1_1), w_1_2)))
    
    loss_total = loss_1 + loss_2 + lam * loss_w

    var_list_0 = tf.trainable_variables('F_0')
    var_list_1_2 = tf.trainable_variables('F_1_2')
    var_list_all = var_list_0 + var_list_1_2
    
    # Without Gradient Clipping
    train_step = self.training(loss_total, learning_rate, var_list_all)
    
    # With Gradient Clipping
    #train_step = self.training_clipped(loss_total, learning_rate, 0.1, var_list_all)
    
    acc_1 =  self.accuracy(probs_1, y_1)
    acc_2 =  self.accuracy(probs_2, y_2)
    
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()

    with tf.Session() as sess:

      sess.run(init)
      
      history_loss_train = []
      history_loss_test = []
      
      history_acc_train_1 = []
      history_acc_test_1 = []
      history_acc_train_2 = []
      history_acc_test_2 = []

      for i in range(n_iter):
        # Train
        rand_index = np.random.choice(len(images_train_1), size = batch_size)
        x_batch_1 = images_train_1[rand_index]
        y_batch_1 = labels_train_1[rand_index]

        rand_index = np.random.choice(len(images_train_2), size = batch_size)
        x_batch_2 = images_train_2[rand_index]
        y_batch_2 = labels_train_2[rand_index]

        feed_dict = {x_1: x_batch_1, x_2: x_batch_2, y_1: y_batch_1, y_2: y_batch_2, keep_prob: 0.7}

        sess.run(train_step, feed_dict = feed_dict)

        temp_loss = sess.run(loss_total, feed_dict = feed_dict)
        temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
        temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)

        history_loss_train.append(temp_loss)
        history_acc_train_1.append(temp_acc_1)
        history_acc_train_2.append(temp_acc_2)

        if (i + 1) % show_step == 0:
          print ('-' * 15)
          print ('Iteration: ' + str(i + 1) + '  Loss: ' + str(temp_loss) +\
                '  Accuracy_1: ' + str(temp_acc_1) +\
                '  Accuracy_2: ' + str(temp_acc_2))

        # Test
        rand_index = np.random.choice(len(images_test_1), size = batch_size)
        x_batch_1 = images_test_1[rand_index]
        y_batch_1 = labels_test_1[rand_index]

        rand_index = np.random.choice(len(images_test_2), size = batch_size)
        x_batch_2 = images_test_2[rand_index]
        y_batch_2 = labels_test_2[rand_index]

        feed_dict = {x_1: x_batch_1, x_2: x_batch_2, y_1: y_batch_1, y_2: y_batch_2, keep_prob: 0.7}

        temp_loss = sess.run(loss_total, feed_dict = feed_dict)
        temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
        temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)

        history_loss_test.append(temp_loss)
        history_acc_test_1.append(temp_acc_1)
        history_acc_test_2.append(temp_acc_2)

      if is_saving:
        model_path = saver.save(sess, model_path)
        print ('done saving at ', model_path)

      print ('-'* 15)    
      fig = plt.figure(figsize = (10, 7))
      ax1 = fig.add_subplot(2, 2, 1)
      ax1.plot(range(n_iter), history_loss_train, 'b-', label = 'Train')
      ax1.plot(range(n_iter), history_loss_test, 'r--', label = 'Test')
      ax1.set_title('Loss')
      ax1.legend(loc = 'upper right')

      ax2 = fig.add_subplot(2, 2, 3)
      ax2.plot(range(n_iter), history_acc_train_1, 'b-', label = 'Train')
      ax2.plot(range(n_iter), history_acc_test_1, 'r--', label = 'Test')
      ax2.set_ylim(0.0, 1.0)
      ax2.set_title('Accuracy_1')
      ax2.legend(loc = 'lower right')

      ax2 = fig.add_subplot(2, 2, 4)
      ax2.plot(range(n_iter), history_acc_train_2, 'b-', label = 'Train')
      ax2.plot(range(n_iter), history_acc_test_2, 'r--', label = 'Test')
      ax2.set_ylim(0.0, 1.0)
      ax2.set_title('Accuracy_2')
      ax2.legend(loc = 'lower right')

      plt.show()
      
  def fit_t(self, images_train, labels_train, images_test, labels_test, \
                filter_size, n_filters_1, n_filters_2, n_units_0, n_units_t, \
                learning_rate, n_iter, batch_size, show_step, is_saving, model_path):

    tf.reset_default_graph()

    x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)

    feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = False)
    
    logits = self.F_t(feat, n_units_0, n_units_t, keep_prob, reuse = False)
    probs = tf.nn.softmax(logits)
    loss = self.loss_cross_entropy(probs, y)  # labeled data
    #loss = self.loss_entropy(probs)  # unlabeled data

    var_list_0 = tf.trainable_variables('F_0')
    var_list_t = tf.trainable_variables('F_t')

    var_list = var_list_0 + var_list_t
    
    # Without Gradient Clipping
    train_step = self.training(loss, learning_rate, var_list)

    # With Gradient Clipping
    #train_step = self.training_clipped(loss, learning_rate, 0.1, var_list)

    acc =  self.accuracy(probs, y)

    init = tf.global_variables_initializer()
    saver = tf.train.Saver()

    with tf.Session() as sess:

      sess.run(init)

      history_loss_train = []
      history_loss_test = []
      history_acc_train = []
      history_acc_test = []

      for i in range(n_iter):
        # for 0 and t 
        # Train
        rand_index = np.random.choice(len(images_train), size = batch_size)
        x_batch = images_train[rand_index]
        y_batch = labels_train[rand_index]

        feed_dict = {x: x_batch, y: y_batch, keep_prob: 0.7}

        sess.run(train_step, feed_dict = feed_dict)

        temp_loss = sess.run(loss, feed_dict = feed_dict)
        temp_acc = sess.run(acc, feed_dict = feed_dict)

        history_loss_train.append(temp_loss)
        history_acc_train.append(temp_acc)

        if (i + 1) % show_step == 0:
          print ('-' * 15)
          print ('Iteration: ' + str(i + 1) + '  Loss_t: ' + str(temp_loss) + \
                '  Accuracy_t: ' + str(temp_acc))

        # Test
        rand_index = np.random.choice(len(images_test), size = batch_size)
        x_batch = images_test[rand_index]
        y_batch = labels_test[rand_index]

        feed_dict = {x: x_batch, y: y_batch, keep_prob: 1.0}

        temp_loss = sess.run(loss, feed_dict = feed_dict)
        temp_acc = sess.run(acc, feed_dict = feed_dict)

        history_loss_test.append(temp_loss)
        history_acc_test.append(temp_acc)

      if is_saving:
        model_path = saver.save(sess, model_path)
        print ('done saving at ', model_path)

      print ('-'* 15)
      fig = plt.figure(figsize = (10, 3))
      ax1 = fig.add_subplot(1, 2, 1)
      ax1.plot(range(n_iter), history_loss_train, 'b-', label = 'Train')
      ax1.plot(range(n_iter), history_loss_test, 'r--', label = 'Test')
      ax1.set_title('Loss_t')
      ax1.legend(loc = 'upper right')

      ax2 = fig.add_subplot(1, 2, 2)
      ax2.plot(range(n_iter), history_acc_train, 'b-', label = 'Train')
      ax2.plot(range(n_iter), history_acc_test, 'r--', label = 'Test')
      ax2.set_ylim(0.0, 1.0)
      ax2.set_title('Accuracy_t')
      ax2.legend(loc = 'lower right')

      plt.show()

  # TDA
  def fit_tda(self, images_train_1, labels_train_1, images_test_1, labels_test_1, \
              images_train_2, labels_train_2, images_test_2, labels_test_2, \
              images_train_t, labels_train_t, images_test_t, labels_test_t, \
              filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, n_units_t, lam, \
              learning_rate, n_iter, batch_size, show_step, k_max, show_step_k, is_saving, model_path):

    tf.reset_default_graph()

    x_1 = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    x_2 = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    x_t = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)

    y_1 = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    y_2 = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    y_t = tf.placeholder(shape = [None, 10], dtype = tf.float32)

    m = tf.placeholder(shape = [None, 1], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)

    feat_1 = self.F_0(x_1, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = False)
    feat_2 = self.F_0(x_2, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = True)
    feat_t = self.F_0(x_t, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = True)
    
    logits_1, logits_2, w_1_1, w_1_2 = self.F_1_2(feat_1, feat_2, n_units_0, n_units_1, keep_prob, reuse = False)
    logits_t = self.F_t(feat_t, n_units_0, n_units_t, keep_prob, reuse = False)
    
    probs_1 = tf.nn.softmax(logits_1)
    loss_1 = self.loss_cross_entropy(probs_1, y_1)
    loss_1_mask = self.loss_cross_entropy_mask(probs_1, y_1, m)
    probs_2 = tf.nn.softmax(logits_2)
    loss_2 = self.loss_cross_entropy(probs_2, y_2)  
    loss_2_mask = self.loss_cross_entropy_mask(probs_2, y_2, m)
    loss_w = tf.reduce_sum(tf.abs(tf.matmul(tf.transpose(w_1_1), w_1_2)))
    loss_1_2 = loss_1 + loss_2 + lam * loss_w
    loss_1_2_mask = loss_1_mask + loss_2_mask + lam * loss_w

    probs_t = tf.nn.softmax(logits_t)
    loss_t = self.loss_cross_entropy(probs_t, y_t)  
    loss_t_mask = self.loss_cross_entropy_mask(probs_t, y_t, m)
    
    var_list_0 = tf.trainable_variables('F_0')
    var_list_1_2 = tf.trainable_variables('F_1_2')
    var_list_t = tf.trainable_variables('F_t')
    var_list_0_1_2 = var_list_0 + var_list_1_2
    var_list_0_t = var_list_0 + var_list_t
    
    # Without Gradient Clipping
    train_step_0_1_2 = self.training(loss_1_2, learning_rate, var_list_0_1_2)
    train_step_0_1_2_mask = self.training(loss_1_2_mask, learning_rate, var_list_0_1_2)
    train_step_0_t = self.training(loss_t, learning_rate, var_list_0_t)
    train_step_0_t_mask = self.training(loss_t_mask, learning_rate, var_list_0_t)
    
    acc_1 =  self.accuracy(probs_1, y_1)
    acc_2 =  self.accuracy(probs_2, y_2)
    acc_t =  self.accuracy(probs_t, y_t)
    
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()

    with tf.Session() as sess:

      sess.run(init)
      
      history_loss_train_1_2 = []
      history_loss_test_1_2 = []
      history_loss_train_t = []
      history_loss_test_t = []
      
      history_acc_train_1 = []
      history_acc_test_1 = []
      history_acc_train_2 = []
      history_acc_test_2 = []
      history_acc_train_t = []
      history_acc_test_t = []

      ##############################
      # Initial training for 1 and 2
      print ('Initial learning')
      for i in range(n_iter):
        rand_index = np.random.choice(len(images_train_1), size = batch_size)
        x_batch_1 = images_train_1[rand_index]
        y_batch_1 = labels_train_1[rand_index]

        rand_index = np.random.choice(len(images_train_2), size = batch_size)
        x_batch_2 = images_train_2[rand_index]
        y_batch_2 = labels_train_2[rand_index]

        feed_dict = {x_1: x_batch_1, x_2: x_batch_2, y_1: y_batch_1, y_2: y_batch_2, keep_prob: 1.0}

        sess.run(train_step_0_1_2, feed_dict = feed_dict)

        temp_loss = sess.run(loss_1_2, feed_dict = feed_dict)
        temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
        temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)

        #history_loss_train_1_2.append(temp_loss)
        #history_acc_train_1.append(temp_acc_1)
        #history_acc_train_2.append(temp_acc_2)

        #if (i + 1) % show_step == 0:
        #  print ('-' * 15)
        #  print ('Iteration: ' + str(i + 1) + '  Loss_1_2: ' + str(temp_loss) +\
        #        '  Accuracy_1: ' + str(temp_acc_1) +\
        #        '  Accuracy_2: ' + str(temp_acc_2))

        # Test
        #rand_index = np.random.choice(len(images_test_1), size = batch_size)
        #x_batch_1 = images_test_1[rand_index]
        #y_batch_1 = labels_test_1[rand_index]

        #rand_index = np.random.choice(len(images_test_2), size = batch_size)
        #x_batch_2 = images_test_2[rand_index]
        #y_batch_2 = labels_test_2[rand_index]

        #feed_dict = {x_1: x_batch_1, x_2: x_batch_2, y_1: y_batch_1, y_2: y_batch_2, keep_prob: 0.7}

        #temp_loss = sess.run(loss_1_2, feed_dict = feed_dict)
        #temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
        #temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)

        #history_loss_test_1_2.append(temp_loss)
        #history_acc_test_1.append(temp_acc_1)
        #history_acc_test_2.append(temp_acc_2)
      
      # Initial training for t
      for i in range(n_iter):
        rand_index = np.random.choice(len(images_train_1), size = batch_size // 2)
        x_batch_1 = images_train_1[rand_index]
        y_batch_1 = labels_train_1[rand_index]

        rand_index = np.random.choice(len(images_train_2), size = batch_size // 2)
        x_batch_2 = images_train_2[rand_index]
        y_batch_2 = labels_train_2[rand_index]

        x_batch = np.concatenate([x_batch_1, x_batch_2], axis = 0)
        y_batch = np.concatenate([y_batch_1, y_batch_2], axis = 0)
        
        feed_dict = {x_t: x_batch, y_t: y_batch, keep_prob: 1.0}
        
        sess.run(train_step_0_t, feed_dict = feed_dict)

        temp_loss = sess.run(loss_t, feed_dict = feed_dict)
        temp_acc = sess.run(acc_t, feed_dict = feed_dict)
                        
        #if (i + 1) % show_step == 0:
        #  print ('-' * 15)
        #  print ('Iteration: ' + str(i + 1) + '  Loss_t: ' + str(temp_loss) +\
        #        '  Accuracy_t: ' + str(temp_acc))
          
      ##############################
      print ()
      print ('-' * 50)
      print ('Domain adaptation')
      
      history_agreement_rate = []
      history_acc_1 = []
      history_acc_2 = []
      history_acc_t = []
      history_mask_t = []
      
      for k in range(k_max):
        rand_index = np.random.choice(len(images_train_t), size = batch_size)
        x_batch_t = images_train_t[rand_index]
        y_batch_t = labels_train_t[rand_index]

        feed_dict = {x_1: x_batch_t, x_2: x_batch_t, y_1: y_batch_t, y_2: y_batch_t, keep_prob: 1.0}

        p_1 = sess.run(probs_1, feed_dict = feed_dict)
        p_2 = sess.run(probs_2, feed_dict = feed_dict)
        p_ave = 0.5 * (p_1 + p_2)

        agreement = np.equal(np.argmax(p_1, axis = 1), np.argmax(p_2, axis = 1)).astype(np.float32)
        agreement_rate = np.mean(agreement)
        history_agreement_rate.append(agreement_rate)
        mask_t = np.reshape(agreement, [-1, 1])
        history_mask_t.append(mask_t)
        
        for i in range(n_iter):
          # for 1 and 2
          rand_index = np.random.choice(len(images_train_1), size = batch_size)
          x_batch_1 = images_train_1[rand_index]
          y_batch_1 = labels_train_1[rand_index]

          rand_index = np.random.choice(len(images_train_2), size = batch_size)
          x_batch_2 = images_train_2[rand_index]
          y_batch_2 = labels_train_2[rand_index]
          
          x_batch_1_t = np.concatenate([x_batch_1, x_batch_t], axis = 0)
          y_batch_1_t = np.concatenate([y_batch_1, p_ave], axis = 0)
          x_batch_2_t = np.concatenate([x_batch_2, x_batch_t], axis = 0)
          y_batch_2_t = np.concatenate([y_batch_2, p_ave], axis = 0)
          
          mask_1_2 = np.concatenate([np.ones(shape = [batch_size, 1], dtype = np.float32), \
                                 np.reshape(agreement, [-1, 1])], axis = 0)
          
          feed_dict = {x_1: x_batch_1_t, x_2: x_batch_2_t, y_1: y_batch_1_t, y_2: y_batch_2_t, \
                       keep_prob: 1.0, m: mask_1_2}

          sess.run(train_step_0_1_2_mask, feed_dict = feed_dict)

          temp_loss = sess.run(loss_1_2_mask, feed_dict = feed_dict)
          temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
          temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)

          #history_loss_train_1_2.append(temp_loss)
          #history_acc_train_1.append(temp_acc_1)
          #history_acc_train_2.append(temp_acc_2)

          #if (i + 1) % show_step == 0:
          #  print ('-' * 15)
          #  print ('Iteration: ' + str(i + 1) + '  Loss_1_2: ' + str(temp_loss) + \
          #        '  Accuracy_1: ' + str(temp_acc_1) +\
          #        '  Accuracy_2: ' + str(temp_acc_2))
          
          # for t
          feed_dict = {x_t: x_batch_t, y_t: p_ave, keep_prob: 1.0, m: mask_t}

          sess.run(train_step_0_t_mask, feed_dict = feed_dict)

          temp_loss = sess.run(loss_t, feed_dict = feed_dict)
          temp_acc = sess.run(acc_t, feed_dict = feed_dict)
          
          #if (i + 1) % show_step == 0:
          #  print ('-' * 15)
          #  print ('Iteration: ' + str(i + 1) + '  Loss_t: ' + str(temp_loss) + \
          #        '  Accuracy_t: ' + str(temp_acc))
            
        rand_index = np.random.choice(len(images_test_t), size = batch_size)
        x_batch_t = images_test_t[rand_index]
        y_batch_t = labels_test_t[rand_index]        
        
        feed_dict = {x_1: x_batch_t, x_2: x_batch_t, y_1: y_batch_t, y_2: y_batch_t, \
                     x_t: x_batch_t, y_t: y_batch_t, keep_prob: 1.0}

        temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
        temp_acc_2 = sess.run(acc_2, feed_dict = feed_dict)
        temp_acc_t = sess.run(acc_t, feed_dict = feed_dict)
        
        history_acc_1.append(temp_acc_1)
        history_acc_2.append(temp_acc_2)
        history_acc_t.append(temp_acc_t)
        
        if (k + 1) % show_step_k == 0:
          print ('-' * 15)
          print ('Adaptation {}'.format(k + 1))
          print ('agreement rate: {:.4f}'.format(agreement_rate))
          print ('Accuracy_1: ' + str(temp_acc_1) + '  Accuracy_2: ' + str(temp_acc_2) +\
                '  Accuracy_t: ' + str(temp_acc_t))
        
      if is_saving:
        model_path = saver.save(sess, model_path)
        print ('done saving at ', model_path)
        
      print ('-'* 15)    
      fig = plt.figure(figsize = (10, 3))
      ax1 = fig.add_subplot(1, 2, 1)
      ax1.plot(history_agreement_rate, 'b-', label = 'Test')
      ax1.set_title('Agreement Rate')
      ax1.legend(loc = 'upper right')

      ax2 = fig.add_subplot(1, 2, 2)
      ax2.plot(range(k_max), history_acc_1, 'b-', label = 'Test_1')
      ax2.plot(range(k_max), history_acc_2, 'r--', label = 'Test_2')
      ax2.plot(range(k_max), history_acc_t, 'y--', label = 'Test_t')
      ax2.set_ylim(0.0, 1.0)
      ax2.set_title('Accuracy')
      ax2.legend(loc = 'lower right')

      plt.show()
            
      return history_agreement_rate, history_acc_1, history_acc_2, history_acc_t, history_mask_t 
      
  def predict_1_2(self, images, labels, filter_size, n_filters_1, n_filters_2, n_units_0, n_units_t, model_path):
    
    x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
    
    feat_1 = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = True)
    feat_2 = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = True)
    
    logits_1, logits_2, w_1_1, w_1_2 = self.F_1_2(feat_1, feat_2, n_units_0, n_units_1, keep_prob, reuse = True)
    probs_1 = tf.nn.softmax(logits_1)
    acc_1 =  self.accuracy(probs_1, y)
    probs_2 = tf.nn.softmax(logits_2)
    acc_2 =  self.accuracy(probs_2, y)
    
    saver = tf.train.Saver()

    with tf.Session() as sess:
      
      saver.restore(sess, model_path)
      
      feed_dict = {x: images, y: labels, keep_prob: 1.0}
      
      print('Accuracy_1: {:.4f}'.format(sess.run(acc_1, feed_dict = feed_dict)))
      print('Accuracy_2: {:.4f}'.format(sess.run(acc_2, feed_dict = feed_dict)))
      
      return sess.run([probs_1, probs_2, w_1_1, w_1_2], feed_dict = feed_dict)
    
  def predict_t(self, images, labels, filter_size, n_filters_1, n_filters_2, n_units_0, n_units_t, model_path):
    
    x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
    y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
    keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
    
    feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
                      keep_prob, reuse = True)
    
    logits = self.F_t(feat, n_units_0, n_units_t, keep_prob, reuse = True)
    probs = tf.nn.softmax(logits)
    acc =  self.accuracy(probs, y)
    
    saver = tf.train.Saver()

    with tf.Session() as sess:
      
      saver.restore(sess, model_path)
      
      feed_dict = {x: images, y: labels, keep_prob: 1.0}
      
      print('Accuracy_t: {:.4f}'.format(sess.run(acc, feed_dict = feed_dict)))
      
      return sess.run(probs, feed_dict = feed_dict)
```

##Parameters
```python
filter_size = 3
n_filters_1 = 16
n_filters_2 = 16
n_units_0 = 128
n_units_1 = 64
n_units_t = 64
lam = 0.0
learning_rate = 0.01
n_iter = 300
batch_size = 64
show_step = 100
model_path = 'datalab/model'
```

##Output
```python
images_train_1 = images_train_0
labels_train_1 = labels_train_0
images_test_1 = images_test_60
labels_test_1 = labels_test_60

images_train_2 = images_train_90
labels_train_2 = labels_train_90
images_test_2 = images_test_60
labels_test_2 = labels_test_60

images_train_t = images_train_60
labels_train_t = labels_train_60
images_test_t = images_test_60
labels_test_t = labels_test_60

test = tda.fit_tda(images_train_1, labels_train_1, images_test_1, labels_test_1, \
            images_train_2, labels_train_2, images_test_2, labels_test_2, \
            images_train_t, labels_train_t, images_test_t, labels_test_t, \
            filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, n_units_t, lam, \
            learning_rate, n_iter, batch_size, show_step, is_saving, model_path)
```

![image.png](https://qiita-image-store.s3.amazonaws.com/0/254604/987a2fe4-7796-4e43-f83b-16b7752f0cbf.png)