Please integrate the PPO agent and the training loop with proper exact synchronization between all the functions and variables in the below given code. provide the 'next_value', The value estimate for the state following the last state in the episode from the training loop appropriately into the compute_gae.

def rearrange_action_output(raw_action_tensor):
    # Ensure raw_action_tensor is in the correct shape, e.g., [13]
    # Rearrange indices to match the specified order
    rearranged_indices = [1, 2, 4, 6, 7, 8, 9, 0, 3, 5, 11, 10, 12]  # Corresponds to [‘l1_value’, ‘l3_value’, ‘l5_value’, ‘w6_value’, ‘l6_value’, ‘w7_value’, ‘l7_value’, ‘w1_value’, ‘w3_value’, ‘w5_value’, ‘Ib_value’, ‘Cc_value’, ‘Vc_value’]
    return raw_action_tensor[rearranged_indices]

class PPOAgent:
    def __init__(self, actor, critic, gnn_model, action_dim, bounds_low, bounds_high,
                 lr_actor=3e-4, lr_critic=1e-3, gamma=0.99, lamda=0.95, epsilon=0.2, std=0.0):
        self.actor = actor(gnn_model.conv2.out_channels, action_dim, std)  # Initialize actor
        self.critic = critic(gnn_model.conv2.out_channels)  # Initialize critic
        self.gnn_model = gnn_model  # GNN model instance
        self.optimizer_actor = optim.Adam(self.actor.parameters(), lr=lr_actor)
        self.optimizer_critic = optim.Adam(self.critic.parameters(), lr=lr_critic)
        self.gamma = gamma
        self.lamda = lamda
        self.epsilon = epsilon
        #self.bounds_low = torch.tensor(bounds_low).float()
        #self.bounds_high = torch.tensor(bounds_high).float()
        self.bounds_low = bounds_low
        self.bounds_high = bounds_high
        self.std = std
        self.epochs = 10  # Define the number of epochs for policy update

    def select_action(self, state, edge_index, edge_attr):
        # Pass state through the GNN model first to get state’s embedding
        state_embedding = self.gnn_model(state, edge_index, edge_attr)
        
        # Then, pass the state embedding through the actor network to get action mean and std
        mean, std = self.actor(state_embedding)

        # Rearranging based on the specifications
        rearranged_mean = rearrange_action_output(mean)

        # Scale mean based on action bounds defined
        mean = self.bounds_low + (torch.sigmoid(rearranged_mean) * (self.bounds_high - self.bounds_low))
        dist = Normal(mean, std)
        
        # Sample an action from the distribution and calculate its log probability
        action = dist.sample()

        action = torch.clamp(action, self.bounds_low, self.bounds_high)  # Ensure action is within bounds
        action_log_prob = dist.log_prob(action)
        
        return action.detach(), action_log_prob.detach()
    
    def compute_gae(self, next_value, rewards, masks, values, gamma=0.99, lambda_=0.95):
        values = values + [next_value]
        gae = 0
        returns = []
        for step in reversed(range(len(rewards))):
            delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step]
            gae = delta + gamma * lambda_ * masks[step] * gae
            returns.insert(0, gae + values[step])
        return returns
    
    def update_policy(self, states, actions, log_probs, returns, advantages):
        for epoch in range(self.epochs):
            sampler = BatchSampler(SubsetRandomSampler(range(len(states))), batch_size=64, drop_last=True)
            for indices in sampler:
                sampled_states = torch.stack([states[i] for i in indices])
                sampled_actions = torch.stack([actions[i] for i in indices])
                sampled_log_probs = torch.stack([log_probs[i] for i in indices])
                sampled_returns = torch.stack([returns[i] for i in indices])
                sampled_advantages = torch.stack([advantages[i] for i in indices])

                # Assuming the actor model returns a distribution from which log probabilities can be computed
                mean, new_std = self.actor(sampled_states)
                dist = Normal(mean, new_std.exp())
                new_log_probs = dist.log_prob(sampled_actions)
                
                ratio = (new_log_probs - sampled_log_probs).exp()

                surr1 = ratio * sampled_advantages
                surr2 = torch.clamp(ratio, 1.0 - self.epsilon, 1.0 + self.epsilon) * sampled_advantages
                actor_loss = -torch.min(surr1, surr2).mean()
                critic_loss = F.mse_loss(sampled_returns, self.critic(sampled_states))

                self.optimizer_actor.zero_grad()
                actor_loss.backward()
                self.optimizer_actor.step()

                self.optimizer_critic.zero_grad()
                critic_loss.backward()
                self.optimizer_critic.step()


# Training loop
def train(env, agent, num_episodes, max_timesteps, batch_size, epsilon):
    for episode in range(num_episodes):
        node_features_tensor, edge_feature_tensor, edge_index, performance_metrics = env.reset()
        episode_rewards = []
        states = []
        actions = []
        log_probs = []
        values = []
        dones = []

        for t in range(max_timesteps):
            action, log_prob = agent.select_action((node_features_tensor, edge_feature_tensor, edge_index))
            next_node_features_tensor, next_edge_feature_tensor, next_edge_index, reward, done, previous_metrics = env.step(action)
            episode_rewards.append(reward)
            states.append((node_features_tensor, edge_feature_tensor, edge_index))
            actions.append(action)
            log_probs.append(log_prob)
            values.append(agent.critic(node_features_tensor, edge_feature_tensor, edge_index).item())
            dones.append(done)
            node_features_tensor = next_node_features_tensor
            edge_feature_tensor = next_edge_feature_tensor
            edge_index = next_edge_index

            if done:
                break

        # Compute returns and advantages
        returns, advantages = agent.compute_gae(episode_rewards, values, dones)

        # Update policy and value network
        agent.update_policy(states, actions, log_probs, returns, advantages)

        # Log episode information
        total_reward = sum(episode_rewards)
        print(f"Episode {episode + 1}/{num_episodes}, Total Reward: {total_reward}")

# Initialize environment and agent
bounds_low = np.array([0.18e-6, 0.18e-6, 0.18e-6, 0.18e-6, 0.18e-6, 0.5e-6, 0.5e-6, 0.5e-6, 0.5e-6, 0.5e-6, 15e-6, 0.1e-12, 0.8])
bounds_high = np.array([0.2e-6, 0.2e-6, 0.2e-6, 0.2e-6, 0.2e-6, 50e-6, 50e-6, 50e-6, 50e-6, 50e-6, 30e-6, 10e-12, 1.4])
target_metrics = {
    'Area': (1e-12, 1e-10),
    'PowerDissipation': (0.005, 0.25),
    'SlewRate': (20, 60),
    'Gain': (70, 100),
    'Bandwidth3dB': (30e3, 100e3),
    'UnityGainFreq': (30e6, 100e6),
    'PhaseMargin': (65, 90)
}

# Initializing the GNN Model (EnhancedGNNModelWithSharedParams)
num_node_features =  24# Define based on your graph data
num_edge_features =  11# Define based on your graph data
num_out_features =  19# Define based on the expected output feature size of your GNN

# Initializing the Actor and Critic with the GNN Model
action_dim =  13# Define according to your action space dimensions
actor = Actor(num_out_features, action_dim, std=0.0)
critic = Critic(num_out_features)

# Hyperparameters
lr_actor = 3e-4
lr_critic = 1e-3
gamma = 0.99
lambda_ = 0.95
epsilon = 0.2
std=0.0

netlist_content = get_netlist_content()

#print(netlist_content)

# Create the environment
env = CircuitEnvironment(server_address, username, password, bounds_low, bounds_high, target_metrics, netlist_content)
gnn_model = EnhancedGNNModelWithSharedParams(num_node_features, num_edge_features, num_out_features)
ppo_agent = PPOAgent(actor, critic, gnn_model, action_dim, lr_actor, lr_critic, gamma, lambda_, epsilon, std)

# Train agent
train(env, ppo_agent, env.num_episodes, env.max_timesteps, env.batch_size, env.epsilon)