//  [python]
def train(self):
        num_episode = 1
        # Record rewards over 30 episodes.
        recent_episode_num = 30
        recent_rewards = []
        recent_avg_reward = None
        # Record the best Avg. reward.
        best_avg_reward = -87.0

        self.sess.run(tf.global_variables_initializer())

        while True:
            episode_done = False
            sum_reward_episode = 0

            last_state = self.env.reset()
            last_state = prepro(last_state)
            action = self.env.action_space.sample()
            state, _, _, _ = self.env.step(action)
            state = prepro(state)

            num_rounds = 1
            num_actions = 1
            num_win = 0
            num_lose = 0

            while not episode_done:
                if self.render:
                    self.env.render()
                    
                delta_state = state - last_state
                last_state = state

                # prob = self.network.forward_pass(np.expand_dims(delta_state, axis=0))[0]
                # self.probs.append(prob)
                
                # action = np.random.choice(self.action_size, 1, p=prob / sum(prob))[0]
                act, v_pred = self.Policy.act(states=np.expand_dims(delta_state, axis=0), stochastic=True)

                act = np.asscalar(act)
                v_pred = np.asscalar(v_pred)

                state, reward, episode_done, info = self.env.step(act + 2)

                self.states.append(delta_state)
                self.actions.append(act)
                self.v_preds.append(v_pred)
                self.rewards.append(reward)
                    
                state = prepro(state)
                sum_reward_episode += reward
                num_actions += 1

                # self.remember(delta_state, action, prob, reward)

                if reward != 0:
                    if reward == -1:
                        num_lose += 1
                    elif reward == +1:
                        num_win += 1
                    print ('Round [{:2d}] {:2d}:{:2d}'.format(num_rounds, num_lose, num_win), end='\r')
                    num_rounds += 1

            v_preds_next = self.v_preds[1:] + [0]  # next state of terminate state has 0 state value

            # Get Avg. of recent rewards.
            recent_rewards.append(sum_reward_episode)
            if len(recent_rewards) > recent_episode_num:
                recent_rewards.pop(0)
            recent_avg_reward = sum(recent_rewards) / len(recent_rewards)
            self.history_recent_avg_reward.append(recent_avg_reward)
            print('\rEpisode: {:d}, Actions: {:4d}, reward: {:2.3f}, Avg. reward: {:2.6f}'.format(
                num_episode, num_actions, sum_reward_episode, recent_avg_reward))

            if recent_avg_reward > best_avg_reward:
                print ('[Save Checkpoint] Avg. reward improved from {:2.6f} to {:2.6f}'.format(
                    best_avg_reward, recent_avg_reward))
                best_avg_reward = recent_avg_reward
                self.save_checkpoint()

            if num_episode % self.batch_size == 0:
                # gradients = np.vstack(self.gradients)
                # rewards = np.vstack(self.rewards)
                # rewards = self.discount_rewards(rewards, self.discount_factor)
                # if np.std(rewards) != 0:
                #     rewards = (rewards - np.mean(rewards)) / np.std(rewards)
                # else:
                #     rewards = rewards - np.mean(rewards)
                # gradients *= rewards

                # X = np.vstack([self.states])
                # Y = self.probs + self.learning_rate * np.vstack([gradients])
                # loss = self.network.train(X, Y)
                # self.states, self.probs, self.gradients, self.rewards = [], [], [], []
                gaes = self.PPO.get_gaes(rewards=self.rewards, v_preds=self.v_preds, v_preds_next=v_preds_next)

                states = np.array(self.states).astype(dtype=np.float32)
                actions = np.array(self.actions).astype(dtype=np.int32)
                gaes = np.array(gaes).astype(dtype=np.float32)
                gaes = (gaes - gaes.mean()) / gaes.std()
                rewards = np.array(self.rewards).astype(dtype=np.float32)
                v_preds_next = np.array(v_preds_next).astype(dtype=np.float32)

                self.PPO.assign_policy_parameters()

                self.PPO.train(states=states,
                          actions=actions,
                          gaes=gaes,
                          rewards=rewards,
                          v_preds_next=v_preds_next)

                self.states, self.actions, self.v_preds, self.rewards = [], [], [], []

            if num_episode % self.save_history_period == 0:
                np.save('history_recent_avg_reward.npy', np.array(self.history_recent_avg_reward))
                
            num_episode += 1