import torch
import torch.nn.functional as F
from torch_geometric.nn import GATConv
import numpy as np
import torch.optim as optim
from torch.distributions import MultivariateNormal

class CustomGNN(torch.nn.Module):
    def __init__(self, in_channels, out_channels):
        super(CustomGNN, self).__init__()
        self.gat1 = GATConv(in_channels, 8, heads=8, dropout=0.6)
        self.gat2 = GATConv(8 * 8, out_channels, heads=1, concat=False, dropout=0.6)

        self.component_nodes_indices = torch.tensor([7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], dtype=torch.long)
        
        # Define masks for tuning selective features
        self.m_features_mask = torch.zeros(24, dtype=torch.bool)
        self.m_features_mask[[18, 19]] = True  
        self.c_features_mask = torch.zeros(24, dtype=torch.bool)
        self.c_features_mask[20] = True
        self.i_features_mask = torch.zeros(24, dtype=torch.bool)
        self.i_features_mask[21] = True
        self.v_features_mask = torch.zeros(24, dtype=torch.bool)
        self.v_features_mask[22] = True

    def forward(self, x, edge_index):
        x = F.dropout(x, p=0.6, training=self.training)
        x = F.elu(self.gat1(x, edge_index))
        x = F.dropout(x, p=0.6, training=self.training)
        x = self.gat2(x, edge_index)

        # Synchronize updates for defined node pairs before updating dynamic features
        # Averaging the values for the synchronous pairs for features at indices [18] and [19]
        # Ensure the original_features tensor is prepared for this operation to not alter unrelated features.
        original_features = x.clone()

        # Define synchronous node pairs and their associated feature indices
        sync_pairs = [(7, 8), (9, 10), (11, 14)]  # Indices in self.component_nodes_indices
        features_to_sync = [18, 19]

        # Perform synchronization
        for pair in sync_pairs:
            # Calculate the mean of the paired node features
            avg_features = original_features[[self.component_nodes_indices[pair[0]], self.component_nodes_indices[pair[1]]], :][:, features_to_sync].mean(dim=0)
            # Assign the averaged features back to the original positions for both nodes in the pair
            original_features[self.component_nodes_indices[pair[0]], features_to_sync] = avg_features
            original_features[self.component_nodes_indices[pair[1]], features_to_sync] = avg_features

        # Apply mask and update dynamic features (if there’s any additional logic for individual component node updates)
        dynamic_updates = torch.zeros_like(x)
        
        # Update logic as previous, but now considering synchronization is already handled
        # Note: With the current use-case, dynamic updates remain as initially set.
        # This placeholder exists for cases where further dynamic processing is applied after synchronization.

        # Ensuring static features are kept as is from original_features and only dynamic are updated
        return original_features * (1 - dynamic_updates) + x * dynamic_updates

class Actor(torch.nn.Module):
    def __init__(self, gnn_model):
        super(Actor, self).__init__()
        self.gnn = gnn_model

    def forward(self, state):
        # State contains node_features_tensor, edge_feature_tensor, edge_index
        node_features_tensor, edge_feature_tensor, edge_index = state
        action_probs = self.gnn(node_features_tensor, edge_index)
        return action_probs

class Critic(torch.nn.Module):
    def __init__(self, state_dim):
        super(Critic, self).__init__()
        self.network = torch.nn.Sequential(
            torch.nn.Linear(state_dim, 128),
            torch.nn.ReLU(),
            torch.nn.Linear(128, 1)
        )

    def forward(self, state):
        return self.network(state)

class PPOAgent:
    def __init__(self, gnn_model, state_dim, action_space, lr_actor, lr_critic, gamma, gae_lambda, epsilon, policy_clip, epochs):
        self.gamma = gamma
        self.gae_lambda = gae_lambda
        self.epsilon = epsilon
        self.policy_clip = policy_clip
        self.epochs = epochs
        
        self.actor = Actor(gnn_model)
        self.critic = Critic(state_dim)
        self.optimizer_actor = optim.Adam(self.actor.parameters(), lr=lr_actor)
        self.optimizer_critic = optim.Adam(self.critic.parameters(), lr=lr_critic)
        self.action_space = action_space  # Assume continuous

    def select_action(self, state):
        state_tensor = torch.FloatTensor(state).unsqueeze(0)  # Adjust dimensions as necessary
        action_probs = self.actor(state_tensor)
        cov_mat = torch.diag(action_probs.var()).unsqueeze(0)  # Ensure variances are positive and form a covariance matrix
        dist = MultivariateNormal(action_probs, cov_mat)
        action = dist.sample()
        log_prob = dist.log_prob(action)
        
        return action.numpy().squeeze(), log_prob.item()
    
    def compute_gae(self, next_value, rewards, masks, values):
        values = values + [next_value]
        gae = 0
        returns = []
        for step in reversed(range(len(rewards))):
            delta = rewards[step] + self.gamma * values[step + 1] * masks[step] - values[step]
            gae = delta + self.gamma * self.gae_lambda * masks[step] * gae
            returns.insert(0, gae + values[step])
        return returns

    def update_policy(self, prev_states, prev_actions, prev_log_probs, returns, advantages):
        advantages = torch.tensor(advantages)
        returns = torch.tensor(returns)
        prev_log_probs = torch.tensor(prev_log_probs)
        
        for _ in range(self.epochs):
            log_probs, state_values, entropy = self.evaluate(prev_states, prev_actions)
            ratios = torch.exp(log_probs - prev_log_probs.detach())
            
            advantages = returns - state_values.detach()
            surr1 = ratios * advantages
            surr2 = torch.clamp(ratios, 1-self.policy_clip, 1+self.policy_clip) * advantages
            
            actor_loss = - torch.min(surr1, surr2).mean()
            critic_loss = F.mse_loss(state_values, returns)
            
            self.optimizer_actor.zero_grad()
            actor_loss.backward()
            self.optimizer_actor.step()
            
            self.optimizer_critic.zero_grad()
            critic_loss.backward()
            self.optimizer_critic.step()

    def evaluate(self, states, actions):
        # Replace with actual evaluation logic based on your training loop requirements
        pass

# Create the environment
env = CircuitEnvironment(server_address, username, password, bounds_low, bounds_high, target_metrics, netlist_content)

For the above given code provide me the proper complete RL training loop to be perfectly synchronize with the above PPO agent class and all its functions 'select_action', 'compute_gae', and 'update_policy'. and also provide me the necessory initialization requirements and instantiation requirements in the above code.