//  [python]
def build_model(self):
    """Builds a model."""
    config = self.config
    # If ps_tasks is zero, the local device is used. When using multiple
    # (non-local) replicas, the ReplicaDeviceSetter distributes the variables
    # across the different devices.
    current_step = tf.cast(self.global_step, tf.float32)
    # g_ratio = (1.0 + 2e-5 * tf.maximum((current_step - 100000.0), 0.0))
    # g_ratio = tf.minimum(g_ratio, 4.0)
    self.d_learning_rate = FLAGS.discriminator_learning_rate
    self.g_learning_rate = FLAGS.generator_learning_rate
    # self.g_learning_rate = FLAGS.generator_learning_rate / (1.0 + 2e-5 * tf.cast(self.global_step, tf.float32))
    # self.g_learning_rate = FLAGS.generator_learning_rate / g_ratio
    with tf.device(tf.train.replica_device_setter(config.ps_tasks)):
      self.d_opt = tf.train.AdamOptimizer(
          self.d_learning_rate, beta1=FLAGS.beta1)
      self.g_opt = tf.train.AdamOptimizer(
          self.g_learning_rate, beta1=FLAGS.beta1)
      if config.sync_replicas and config.num_workers > 1:
        self.d_opt = tf.train.SyncReplicasOptimizer(
            opt=self.d_opt, replicas_to_aggregate=config.replicas_to_aggregate)
        self.g_opt = tf.train.SyncReplicasOptimizer(
            opt=self.g_opt, replicas_to_aggregate=config.replicas_to_aggregate)

    if config.num_towers > 1:
      all_d_grads = []
      all_g_grads = []
      for idx, device in enumerate(self.devices):
        with tf.device('/%s' % device):
          with tf.name_scope('device_%s' % idx):
            with ops.variables_on_gpu0():
              self.build_model_single_gpu(
                  gpu_idx=idx,
                  batch_size=config.batch_size,
                  num_towers=config.num_towers)
              d_grads = self.d_opt.compute_gradients(self.d_losses[-1],
                                                     var_list=self.d_vars)
              g_grads = self.g_opt.compute_gradients(self.g_losses[-1],
                                                     var_list=self.g_vars)
              all_d_grads.append(d_grads)
              all_g_grads.append(g_grads)
      d_grads = ops.avg_grads(all_d_grads)
      g_grads = ops.avg_grads(all_g_grads)
    else:
      with tf.device(tf.train.replica_device_setter(config.ps_tasks)):
        # TODO(olganw): reusing virtual batchnorm doesn't work in the multi-
        # replica case.
        self.build_model_single_gpu(batch_size=config.batch_size,
                                    num_towers=config.num_towers)
        d_grads = self.d_opt.compute_gradients(self.d_losses[-1],
                                               var_list=self.d_vars)
        g_grads = self.g_opt.compute_gradients(self.g_losses[-1],
                                               var_list=self.g_vars)
    with tf.device(tf.train.replica_device_setter(config.ps_tasks)):
      update_moving_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
      print('update_moving_ops', update_moving_ops)
      if config.sync_replicas:
        with tf.control_dependencies(update_moving_ops):
          d_step = tf.get_variable('d_step', initializer=0, trainable=False)
          self.d_optim = self.d_opt.apply_gradients(d_grads, global_step=d_step)
          g_step = tf.get_variable('g_step', initializer=0, trainable=False)
          self.g_optim = self.g_opt.apply_gradients(g_grads, global_step=g_step)
      else:
        # Don't create any additional counters, and don't update the global step
        with tf.control_dependencies(update_moving_ops):
          self.d_optim = self.d_opt.apply_gradients(d_grads)
          self.g_optim = self.g_opt.apply_gradients(g_grads)