anything seems off ? :"from . import RLAgent
from common.registry import Registry
import numpy as np
import os
import random
from collections import OrderedDict, deque
import gym

from generator.lane_vehicle import LaneVehicleGenerator
from generator.intersection_phase import IntersectionPhaseGenerator
import torch
from torch import nn
import torch.nn.functional as F
import torch_scatter
import torch.optim as optim
from torch.nn.utils import clip_grad_norm_

from torch_geometric.nn import MessagePassing
from torch_geometric.data import Data, Batch
from torch_geometric.utils import add_self_loops
from torch.nn import TransformerEncoder, TransformerEncoderLayer

import copy

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using device: {device}")
torch.autograd.set_detect_anomaly(True)
@Registry.register_model('TestLight')
class TestLightAgent(RLAgent):
    def __init__(self, world, rank):
        super().__init__(world, world.intersection_ids[rank])
        self.rank = rank  # Initialize the rank attribute
        self.buffer_size = Registry.mapping['trainer_mapping']['setting'].param['buffer_size']
        self.replay_buffer = deque(maxlen=self.buffer_size)

        self.graph = Registry.mapping['world_mapping']['graph_setting'].graph
        self.world = world
        self.sub_agents = len(self.world.intersections)
        self.edge_idx = torch.tensor(self.graph['sparse_adj'].T, dtype=torch.long).to(device)
        
        self.phase = Registry.mapping['model_mapping']['setting'].param['phase']
        self.one_hot = Registry.mapping['model_mapping']['setting'].param['one_hot']
        self.model_dict = Registry.mapping['model_mapping']['setting'].param

        # Initialize intersection importance with ones for all nodes
        self.intersection_importance = {i: 1.0 for i in range(self.sub_agents)}

        observation_generators = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ['lane_count'], in_only=True, average=None)
            observation_generators.append((node_idx, tmp_generator))
        sorted(observation_generators, key=lambda x: x[0])
        self.ob_generator = observation_generators

        rewarding_generators = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ["lane_waiting_count"],
                                                 in_only=True, average='all', negative=True)
            rewarding_generators.append((node_idx, tmp_generator))
        sorted(rewarding_generators, key=lambda x: x[0])
        self.reward_generator = rewarding_generators

        queues = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ["lane_waiting_count"], 
                                                 in_only=True, negative=False)
            queues.append((node_idx, tmp_generator))
        sorted(queues, key=lambda x: x[0])
        self.queue = queues

        delays = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ["lane_delay"], 
                                                 in_only=True, average="all", negative=False)
            delays.append((node_idx, tmp_generator))
        sorted(delays, key=lambda x: x[0])
        self.delay = delays

        phasing_generators = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = IntersectionPhaseGenerator(self.world, inter, ['phase'],
                                                       targets=['cur_phase'], negative=False)
            phasing_generators.append((node_idx, tmp_generator))
        sorted(phasing_generators, key=lambda x: x[0])
        self.phase_generator = phasing_generators

        self.action_space = gym.spaces.Discrete(len(self.world.intersections[0].phases))
        min_ob_length = max([ob[1].ob_length for ob in self.ob_generator])
        self.ob_length = min_ob_length


        self.learning_rate = self.model_dict.get('learning_rate', 1e-4)

        self.target_update_freq = Registry.mapping['trainer_mapping']['setting'].param['target_update_freq']

        self.gamma = self.model_dict.get('gamma', 0.99)
        self.grad_clip = self.model_dict.get('grad_clip', 5.0)
        self.epsilon = self.model_dict.get('epsilon', 1.0)
        self.epsilon_decay = self.model_dict.get('epsilon_decay', 0.99)
        self.epsilon_min = self.model_dict.get('epsilon_min', 0.1)

        self.vehicle_max = self.model_dict.get('vehicle_max', 100)
        self.batch_size = self.model_dict.get('batch_size', 64)

        self.episode_count = 0
        self.update_interval = self.model_dict.get('update_interval', 100)  # Initial interval for updating importance
        self.decay_factor = self.model_dict.get('decay_factor', 1.05)  # Factor to increase the interval after each update
        self.softmax_temperature = self.model_dict.get('softmax_temperature', 1.0)  # Temperature for softmax
        self.softmax_feather = self.model_dict.get('softmax_feather', 1.0)  # Feather for softmax
    
        self.model = self._build_model().to(device)
        self.optimizer = optim.AdamW(self.model.parameters(), lr=self.learning_rate)

        # A3C specific parameters
        self.entropy_coef = self.model_dict.get('entropy_coef', 0.01)
        self.value_loss_coef = self.model_dict.get('value_loss_coef', 0.5)
        self.max_grad_norm = self.model_dict.get('max_grad_norm', 0.5)

    def reset(self):
        observation_generators = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ['lane_count'], in_only=True, average=None)
            observation_generators.append((node_idx, tmp_generator))
        sorted(observation_generators, key=lambda x: x[0])  # now generator's order is according to its index in graph
        self.ob_generator = observation_generators

        #  get reward generator for CoLightAgent
        rewarding_generators = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ["lane_waiting_count"],
                                                 in_only=True, average='all', negative=True)
            rewarding_generators.append((node_idx, tmp_generator))
        sorted(rewarding_generators, key=lambda x: x[0])  # now generator's order is according to its index in graph
        self.reward_generator = rewarding_generators

        #  phase generator
        phasing_generators = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = IntersectionPhaseGenerator(self.world, inter, ['phase'],
                                                       targets=['cur_phase'], negative=False)
            phasing_generators.append((node_idx, tmp_generator))
        sorted(phasing_generators, key=lambda x: x[0])  # now generator's order is according to its index in graph
        self.phase_generator = phasing_generators

        # queue metric
        queues = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ["lane_waiting_count"], 
                                                 in_only=True, negative=False)
            queues.append((node_idx, tmp_generator))
        # now generator's order is according to its index in graph
        sorted(queues, key=lambda x: x[0])
        self.queue = queues

        # delay metric
        delays = []
        for inter in self.world.intersections:
            node_id = inter.id if 'GS_' not in inter.id else inter.id[3:]
            node_idx = self.graph['node_id2idx'][node_id]
            tmp_generator = LaneVehicleGenerator(self.world, inter, ["lane_delay"], 
                                                 in_only=True, average="all", negative=False)
            delays.append((node_idx, tmp_generator))
        # now generator's order is according to its index in graph
        sorted(delays, key=lambda x: x[0])
        self.delay = delays

    def get_ob(self):
        x_obs = []  # sub_agents * lane_nums,
        for i in range(len(self.ob_generator)):
            ob = self.ob_generator[i][1].generate()/ self.vehicle_max
            ob = np.pad(ob, (0, self.ob_length - ob.shape[-1] ))
            x_obs.append(ob)
            
        x_obs = np.array(x_obs, dtype=np.float32)
        return x_obs

    def get_reward(self):
        rewards = []
        for i in range(len(self.reward_generator)):
            rewards.append(self.reward_generator[i][1].generate())
        rewards = np.squeeze(np.array(rewards, dtype=np.float32)) * 12
        return rewards

    def get_phase(self):
        phase = []
        for i in range(len(self.phase_generator)):
            phase.append((self.phase_generator[i][1].generate()))
        phase = (np.concatenate(phase)).astype(np.int8)
        return phase

    def get_queue(self):
        queue = []
        for item in self.queue:
            item = item[1].generate()
            item = np.pad(item, (0, self.ob_length - item.shape[-1]))
            queue.append(item)
            
        tmp_queue = np.squeeze(np.array(queue, dtype=np.float32))
        queue = np.sum(tmp_queue, axis=1 if len(tmp_queue.shape)==2 else 0)
        return queue

    def get_delay(self):
        delay = []
        for i in range(len(self.delay)):
            delay.append((self.delay[i][1].generate()))
        delay = np.squeeze(np.array(delay, dtype=np.float32))
        return delay

    def get_action(self, ob, phase, test=False):
        if not test and np.random.rand() <= self.epsilon:
            return self.sample()

        observation = torch.tensor(ob, dtype=torch.float32).to(device)
        edge_index = self.edge_idx.to(device)

        action_probs, state_value, attentions = self.model(x=observation, edge_index=edge_index, train=False)
        action_distribution = torch.distributions.Categorical(logits=action_probs)
        actions = action_distribution.sample()

        # Update intersection importance using attentions
        self.update_intersection_importance(attentions)

        return actions.cpu().numpy()

    def sample(self):
        return np.random.randint(0, self.action_space.n, self.sub_agents)

    def update_intersection_importance(self, attentions):
        # Use attentions from the upper transformer
        upper_attentions = attentions.get('upper_transformer', [])
        if upper_attentions:
            # Collect and process the attentions
            # Each attention matrix is of shape [num_heads, batch_size, seq_len, seq_len]
            # Since seq_len = batch_size in our case, and seq_len may be greater than 1
            # We need to process attentions accordingly

            # For simplicity, compute the mean attention scores over heads and layers
            importance_scores = None
            for attn_weights in upper_attentions:
                # attn_weights shape: [batch_size, num_heads, seq_len, seq_len]
                attn_mean = attn_weights.mean(dim=1).squeeze()  # Average over heads
                attn_mean = attn_mean.diagonal(dim1=-2, dim2=-1)  # Get self-attention scores
                if importance_scores is None:
                    importance_scores = attn_mean
                else:
                    importance_scores += attn_mean
            importance_scores = importance_scores / len(upper_attentions)  # Average over layers

            # Update intersection importance
            for i, score in enumerate(importance_scores):
                self.intersection_importance[i] = score.item()

            # Apply softmax with temperature
            importance_tensor = torch.tensor([self.intersection_importance[i] for i in range(self.sub_agents)], device=device)
            temperature = self.softmax_temperature
            normalized_importance_tensor = F.softmax(importance_tensor / temperature, dim=0)
            normalized_importance = {i: normalized_importance_tensor[i].item() for i in range(self.sub_agents)}

            self.intersection_importance = normalized_importance
            self.model.intersection_importance = normalized_importance

    def _build_model(self):
        model = TestLightNet(
            self.ob_length, 
            self.action_space.n, 
            self.intersection_importance, 
            softmax_temperature=self.softmax_temperature,
            softmax_feather=self.softmax_feather,
            update_interval=self.update_interval,
            decay_factor=self.decay_factor,
            **self.model_dict
        )
        return model

    def remember(self, last_obs, last_phase, actions, actions_prob, rewards, obs, cur_phase, done, key):
        self.replay_buffer.append((key, (last_obs, last_phase, actions, rewards, obs, cur_phase)))

    def _batchwise(self, samples):
        batch_list = []
        batch_list_p = []
        actions = []
        rewards = []
        for item in samples:
            dp = item[1]
            state = torch.tensor(dp[0], dtype=torch.float32)
            batch_list.append(Data(x=state, edge_index=self.edge_idx))

            state_p = torch.tensor(dp[4], dtype=torch.float32)
            batch_list_p.append(Data(x=state_p, edge_index=self.edge_idx))
            rewards.append(dp[3])
            actions.append(dp[2])
        batch_t = Batch.from_data_list(batch_list)
        batch_tp = Batch.from_data_list(batch_list_p)
        rewards = torch.tensor(np.array(rewards), dtype=torch.float32)
        actions = torch.tensor(np.array(actions), dtype=torch.long)
        if self.sub_agents > 1:
            rewards = rewards.view(rewards.shape[0] * rewards.shape[1])
            actions = actions.view(actions.shape[0] * actions.shape[1])
        return batch_t, batch_tp, rewards, actions

    def train(self):
        if len(self.replay_buffer) < self.batch_size:
            return 0

        samples = random.sample(self.replay_buffer, self.batch_size)

        # Prepare batches
        observations = []
        actions = []
        rewards = []
        next_observations = []
        dones = []

        for item in samples:
            _, (last_obs, _, action, reward, obs, _) = item
            observations.append(last_obs)
            actions.append(action)
            rewards.append(reward)
            next_observations.append(obs)
            dones.append(False)  # Assuming 'done' is always False; adjust if necessary

        observations = torch.tensor(np.array(observations), dtype=torch.float32).to(device)
        actions = torch.tensor(np.array(actions), dtype=torch.long).to(device)
        rewards = torch.tensor(np.array(rewards), dtype=torch.float32).to(device)
        next_observations = torch.tensor(np.array(next_observations), dtype=torch.float32).to(device)
        dones = torch.tensor(dones, dtype=torch.float32).to(device)

        # Flatten the tensors if necessary
        observations = observations.view(-1, observations.shape[-1])
        actions = actions.view(-1)
        rewards = rewards.view(-1)
        next_observations = next_observations.view(-1, next_observations.shape[-1])

        # Compute current policy and value
        action_probs, state_values, attentions = self.model(x=observations, edge_index=self.edge_idx.to(device))
        action_distribution = torch.distributions.Categorical(logits=action_probs)
        log_probs = action_distribution.log_prob(actions)
        entropy = action_distribution.entropy().mean()

        # Compute next state values
        with torch.no_grad():
            _, next_state_values, _ = self.model(x=next_observations, edge_index=self.edge_idx.to(device))

        # Compute targets and advantages
        targets = rewards + self.gamma * next_state_values.squeeze(1) * (1 - dones)
        advantages = targets - state_values.squeeze(1)

        # Compute losses
        value_loss = advantages.pow(2).mean()
        policy_loss = -(log_probs * advantages.detach()).mean()
        entropy_loss = -self.entropy_coef * entropy

        total_loss = policy_loss + self.value_loss_coef * value_loss + entropy_loss

        # Optimize the model
        self.optimizer.zero_grad()
        total_loss.backward()
        torch.nn.utils.clip_grad_norm(self.model.parameters(), self.max_grad_norm)
        self.optimizer.step()

        # Update epsilon if using epsilon-greedy (though typically not used in actor-critic methods)
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay

        return total_loss.item()

    def update_target_network(self):
        weights = self.model.state_dict()
        self.target_model.load_state_dict(weights)

    def load_model(self, e):
        model_name = os.path.join(Registry.mapping['logger_mapping']['path'].path,
                                  'model', f'{e}_{self.rank}.pt')        
        self.model = self._build_model()
        self.model.load_state_dict(torch.load(model_name))
        self.target_model = self._build_model()
        self.target_model.load_state_dict(torch.load(model_name))

    def save_model(self, e):
        path = os.path.join(Registry.mapping['logger_mapping']['path'].path, 'model')
        if not os.path.exists(path):
            os.makedirs(path)
        model_name = os.path.join(path, f'{e}_{self.rank}.pt')
        torch.save(self.target_model.state_dict(), model_name)


class TestLightNet(nn.Module):
    def __init__(self, input_dim, output_dim, intersection_importance, **kwargs):
        super(TestLightNet, self).__init__()
        self.model_dict = kwargs
        self.action_space = gym.spaces.Discrete(output_dim)
        self.features = input_dim
        self.intersection_importance = intersection_importance
        self.softmax_temperature = kwargs.get('softmax_temperature', 1.0)
        self.softmax_feather = kwargs.get('softmax_feather', 1.0)
        self.update_interval = kwargs.get('update_interval', 100)
        self.decay_factor = kwargs.get('decay_factor', 1.05)
        self.embedding_MLP = Embedding_MLP(self.features, layers=self.model_dict.get('NODE_EMB_DIM'))
        
        self.embedding_output_dim = self.model_dict.get('NODE_EMB_DIM')[-1]
        
        self.num_gat_layers = self.model_dict.get('N_LAYERS', 5)
        self.num_heads = self.model_dict.get('NUM_HEADS', [1])[0]
        
        self.head_dim = self.embedding_output_dim // self.num_heads
        
        self.gat_layers = nn.ModuleList([
            MultiHeadAttModel(d=self.embedding_output_dim, dv=self.head_dim, d_out=self.embedding_output_dim, nv=self.num_heads, suffix=f'layer_{i}')
            for i in range(self.num_gat_layers)
        ])
        
        self.skip_connections = nn.ModuleList([
            nn.Linear(self.embedding_output_dim, self.embedding_output_dim)
            for _ in range(self.num_gat_layers)
        ])

        self.spatial_attention = nn.Parameter(torch.Tensor(1, self.num_heads, self.embedding_output_dim))
        self.temporal_attention = nn.Parameter(torch.Tensor(1, self.num_heads, self.embedding_output_dim))

        # Lower Transformer Encoder
        self.lower_transformer_layer = CustomTransformerEncoderLayer(
            d_model=self.embedding_output_dim, 
            nhead=self.num_heads,
            dropout=kwargs.get('dropout', 0.1)
        )
        self.lower_transformer = CustomTransformerEncoder(
            self.lower_transformer_layer, 
            num_layers=1
        )

        # Upper Transformer Encoder
        self.upper_transformer_layer = CustomTransformerEncoderLayer(
            d_model=self.embedding_output_dim, 
            nhead=self.num_heads,
            dropout=kwargs.get('dropout', 0.1)
        )
        self.upper_transformer = CustomTransformerEncoder(
            self.upper_transformer_layer, 
            num_layers=1
        )

        # Actor-Critic Networks
        self.actor = nn.Sequential(
            nn.Linear(self.embedding_output_dim, 128),
            nn.ReLU(),
            nn.Linear(128, output_dim)
        )
        self.critic = nn.Sequential(
            nn.Linear(self.embedding_output_dim, 128),
            nn.ReLU(),
            nn.Linear(128, 1)
        )

    def forward(self, x, edge_index, train=True):
        # Embedding
        x = self.embedding_MLP(x)

        # Lower Transformer
        x = x.unsqueeze(1)  # Add sequence dimension for Transformer encoder
        x = x.permute(1, 0, 2)  # Permute dimensions for Transformer encoder
        x, lower_attentions = self.lower_transformer(x)
        x = x.permute(1, 0, 2).squeeze(1)  # Permute back and remove sequence dimension

        # GAT Layers with Skip Connections
        gat_attentions = []
        for gat_layer, skip_connection in zip(self.gat_layers, self.skip_connections):
            x_new, att = gat_layer(x, edge_index)
            x = x_new + skip_connection(x)
            gat_attentions.append(att)

        # Upper Transformer
        x = x.unsqueeze(1)  # Add sequence dimension
        x = x.permute(1, 0, 2)  # Permute dimensions
        x, upper_attentions = self.upper_transformer(x)
        x = x.permute(1, 0, 2).squeeze(1)  # Permute back and remove sequence dimension

        # Actor-Critic Outputs
        action_probs = self.actor(x)
        state_value = self.critic(x)

        # Collect all attentions
        attentions = {
            'lower_transformer': lower_attentions,
            'gat_layers': gat_attentions,
            'upper_transformer': upper_attentions
        }

        return action_probs, state_value, attentions

class Embedding_MLP(nn.Module):
    def __init__(self, input_dim, layers):
        super(Embedding_MLP, self).__init__()
        self.layers = nn.ModuleList()
        for i in range(len(layers)):
            self.layers.append(nn.Linear(input_dim if i == 0 else layers[i-1], layers[i]))
            self.layers.append(nn.ReLU())

    def forward(self, x):
        for layer in self.layers:
            x = layer(x)
        return x

class MultiHeadAttModel(MessagePassing):
    def __init__(self, d, dv, d_out, nv, suffix):
        super(MultiHeadAttModel, self).__init__(aggr='add')
        self.nv = nv
        self.d = d
        self.dv = dv
        self.d_out = d_out
        self.suffix = suffix
        self.att = nn.Parameter(torch.Tensor(1, nv, d))
        self.lin = nn.Linear(d, d_out)
        self.reset_parameters()

    def reset_parameters(self):
        nn.init.xavier_uniform_(self.att)
        nn.init.xavier_uniform_(self.lin.weight)

    def forward(self, x, edge_index):
        return self.propagate(edge_index, x=x)

    def message(self, x_j, edge_index, size):
        x_j = x_j.view(-1, self.nv, self.d)
        alpha = (x_j * self.att).sum(dim=-1)
        alpha = F.leaky_relu(alpha, negative_slope=0.2)
        alpha = F.softmax(alpha, dim=1)
        return x_j * alpha.unsqueeze(-1)

    def update(self, aggr_out):
        return self.lin(aggr_out.view(-1, self.nv * self.dv))

class CustomTransformerEncoderLayer(nn.Module):
    def __init__(self, d_model, nhead, **kwargs):
        super(CustomTransformerEncoderLayer, self).__init__()
        self.self_attn = nn.MultiheadAttention(d_model, nhead, **kwargs)
        self.linear1 = nn.Linear(d_model, d_model)
        self.dropout = nn.Dropout(kwargs.get('dropout', 0.1))
        self.linear2 = nn.Linear(d_model, d_model)

        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.dropout1 = nn.Dropout(kwargs.get('dropout', 0.1))
        self.dropout2 = nn.Dropout(kwargs.get('dropout', 0.1))
        self.activation = nn.ReLU()

    def forward(self, src, src_mask=None, src_key_padding_mask=None):
        src2, attn_weights = self.self_attn(
            src, src, src,
            attn_mask=src_mask,
            key_padding_mask=src_key_padding_mask,
            need_weights=True
        )
        src = src + self.dropout1(src2)
        src = self.norm1(src)
        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
        src = src + self.dropout2(src2)
        src = self.norm2(src)
        return src, attn_weights  # Return attention weights

class CustomTransformerEncoder(nn.Module):
    def __init__(self, encoder_layer, num_layers):
        super(CustomTransformerEncoder, self).__init__()
        self.layers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(num_layers)])
        self.num_layers = num_layers
        self.norm = nn.LayerNorm(encoder_layer.linear1.in_features)

    def forward(self, src, mask=None, src_key_padding_mask=None):
        output = src
        attentions = []
        for mod in self.layers:
            output, attn_weights = mod(output, src_mask=mask, src_key_padding_mask=src_key_padding_mask)
            attentions.append(attn_weights)
        output = self.norm(output)
        return output, attentions"