This is my code for RL algorithm, where the algorithm tune the 'action' ['L1', 'L3', 'L5', 'L6', 'L7', 'W1', 'W3', 'W5', 'W6', 'W7', 'Io', 'CP', 'VC'] with in its bounds, to get the desired target['all transistors are in saturation_condition', 'area', 'power dissipation', 'DC gain', 'Slew rate', 'Bandwidth3dB', 'Unity gain bandwidth', 'Phase margin'] with in its predefined range, from my circuit simulator where my two stage opamp circuit is simulated. Now i need to modify the existing implementation with Graph Newral Network with RL algorithm. For my new requirement What should I need to modify/update in the existing code, How should I implement the GNN with RL in the existing code and Where Should I Implement the GNN with RL. Please note: my circuit topology is fixed, i need to only tune the circuit design components values to achieve the target requirement, and utilize the learned knowledge to the unseen graph topology(unseen circuit)

from Simulator_Env1 import CircuitSimulator

class CircuitEnvironment:
    def __init__(self, server_address, username, password, bounds_low, bounds_high, target_metrics): 
    def connect_to_simulator(self):        
    def disconnect_from_simulator(self):        
    def normalize_parameters(self, parameters):        
        return normalized_parameters
    def encode_regions(self, transistor_regions):        
        return encoded_regions
    def encode_saturation_condition(self, saturation_condition):        
            return 0
    def normalize_performance_metrics(self, performance_metrics):        
        return normalized_metrics        
    def reset(self):        
        return state, performance_metrics
    def step(self, raw_actions, previous_metrics):        
        return state, reward, done, previous_metrics
    def is_performance_metrics_in_target(self, metrics):
        return np.all(metrics >= self.PERFORMANCE_METRICS_TARGET_LOW) and np.all(metrics <= self.PERFORMANCE_METRICS_TARGET_HIGH)      
    def calculate_reward(self, c_metrics, p_metrics, transistor_regions, previous_transistor_regions):        
        return reward     
    def check_done(self, performance_metrics, encoded_saturation_condition):        
        return True
    def initialize_actor_critic_networks(self, state_size, action_size, hidden_sizes_actor, hidden_sizes_critic, action_std_init, learning_rate_actor, learning_rate_critic, bounds_low, bounds_high):
        self.actor = ActorNetwork(state_size, action_size, hidden_sizes_actor, bounds_low, bounds_high, action_std_init)
        self.critic = CriticNetwork(state_size, hidden_sizes_critic)
        self.optimizer_actor = optim.Adam(self.actor.parameters(), lr=learning_rate_actor)
        self.optimizer_critic = optim.Adam(self.critic.parameters(), lr=learning_rate_critic)

# Define the Actor and Critic Networks
class ActorNetwork(nn.Module):
    def __init__(self, state_size, action_size, hidden_sizes, bounds_low, bounds_high, action_std_init):

    def forward(self, state):
        return action_mean, log_std
    
    def scale_action(self, action):
        return scaled_action

class CriticNetwork(nn.Module):
    def __init__(self, state_size, hidden_sizes):
    
    def forward(self, state):
        return value

# Select Action Function
def select_action(state, actor, critic, performance_metrics):      
    return action.detach().numpy(), log_prob, value.squeeze().detach().numpy(), performance_metrics

# Generalized Advantage Estimation (GAE)
    return returns, advantages

# Policy and Value Network Update Function
def update_policy(actor, critic, optimizer_actor, optimizer_critic, states, actions, old_log_probs, returns, advantages, batch_size, epsilon):
    # Prepare data for training
    # Create DataLoader    
    # Iterate over batches
    for batch in dataloader:
        batch_states, batch_actions, batch_old_log_probs, batch_returns, batch_advantages = batch
        # Compute new log probabilities and values 
        # Compute PPO loss for actor
        # Compute value loss for critic
        # Update actor and critic      

    
# Constants
# Define the values for server_address, username, password, bounds, and target metrics
# Define the constants and target metrics for reward calculation

# Create the environment
env = CircuitEnvironment(server_address, username, password, bounds_low, bounds_high, target_metrics)

# Initialize Networks and Optimizers

env.initialize_actor_critic_networks(state_size, action_size, hidden_sizes_actor, hidden_sizes_critic, action_std_init, learning_rate_actor, learning_rate_critic, bounds_low, bounds_high)

# Set the Hyperparameters
# Set initial epsilon value and decay rate

# Training Loop
for episode in range(num_episodes):
    # Define the Training Loop 
    state, performance_metrics = env.reset()
        
    # Run one episode
    for t in range(max_timesteps):
        action, log_prob, value, perf_metrics = select_action(state, env.actor, env.critic, performance_metrics)        
        action = np.clip(action, bounds_low, bounds_high)        
        next_state, reward, done, previous_metrics = env.step(action, perf_metrics) # Take action in environment

        performance_metrics = previous_metrics
        state = next_state
        if done:
            break
       
    # Compute returns and advantages    
    # Update policy and value network     
    # Update epsilon   
    
    # Log episode information
    total_reward = sum(episode_rewards)        
env.disconnect_from_simulator()