I have this code:
```py
import os
import random
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import soundfile as sf
from collections import deque
import matplotlib.pyplot as plt
import pygame
import tempfile

from transformers import EncodecModel, AutoProcessor

# Set random seeds for reproducibility
torch.manual_seed(42)
np.random.seed(42)
random.seed(42)

# Device configuration
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

audio_processor = AutoProcessor.from_pretrained("facebook/encodec_24khz")
audio_model = EncodecModel.from_pretrained("facebook/encodec_24khz").to(device)

# Ensure the messages directory exists
os.makedirs("messages", exist_ok=True)

class MazeEnvironment:
    CELL_SIZE = 40  # pixels

    WALL_COLOR = (0, 0, 0)           # Black
    FREE_COLOR = (255, 255, 255)     # White
    GOAL_COLOR = (0, 255, 0)         # Green
    AGENT_COLORS = [(0, 0, 255), (255, 0, 0)]  # Blue, Red

    def __init__(self, maze_size=(21, 21), num_agents=2, visibility_radius=3, render=False):
        self.maze_size = maze_size
        self.num_agents = num_agents
        self.visibility_radius = visibility_radius
        self.rendering = render
        if self.rendering:
            pygame.init()
            self.screen_width = self.maze_size[1] * self.CELL_SIZE
            self.screen_height = self.maze_size[0] * self.CELL_SIZE
            self.screen = pygame.display.set_mode((self.screen_width, self.screen_height))
            pygame.display.set_caption("Maze Environment")
            self.clock = pygame.time.Clock()
        self.agents = []  # Инициализируем пустой список агентов
        self.reset()

    def reset(self):
        # Убедитесь, что размеры лабиринта нечетные
        width, height = self.maze_size
        if width % 2 == 0:
            width += 1
        if height % 2 == 0:
            height += 1
        self.maze = self.generate_maze(width, height)
        print("Generated Maze:")
        self.print_maze()  # Используем улучшенный вывод лабиринта

        # Получение всех свободных клеток
        free_cells = np.argwhere(self.maze == 0)
        num_free_cells = free_cells.shape[0]
        print(f"Number of free cells: {num_free_cells}")  # Debugging

        if num_free_cells < self.num_agents + 1:  # +1 для цели
            raise Exception(f"Not enough free cells to place all agents and the goal. Free cells: {num_free_cells}, Agents: {self.num_agents}")

        # Инициализация позиций агентов
        self.agent_positions = []
        max_attempts = 1000  # Maximum attempts to place an agent
        for agent_idx in range(self.num_agents):
            attempts = 0
            while attempts < max_attempts:
                pos = tuple(random.choice(free_cells))
                if pos not in self.agent_positions:
                    self.agent_positions.append(pos)
                    print(f"Agent {agent_idx} placed at: {pos}")  # Debugging: Print agent positions
                    break
                attempts += 1
            else:
                raise Exception(f"Could not place agent {agent_idx} after {max_attempts} attempts. Not enough free cells.")

        # Установка позиции цели в случайную свободную клетку, отличную от позиций агентов
        remaining_free = [tuple(cell) for cell in free_cells if tuple(cell) not in self.agent_positions]
        if not remaining_free:
            raise Exception("No free cells available to place the goal.")
        self.goal_position = random.choice(remaining_free)
        print(f"Goal position set at: {self.goal_position}")  # Debugging: Print goal position

        # Убедитесь, что позиция цели свободна
        assert self.maze[self.goal_position[0]][self.goal_position[1]] == 0, "Goal placed on a wall"

        # Initialize done flag
        self.done = False

        # Initialize communication buffers (for storing incoming messages)
        self.communication_buffers = [deque(maxlen=5) for _ in range(self.num_agents)]

        # Initialize rewards
        self.rewards = [0 for _ in range(self.num_agents)]

        # Важно: Связываем агентов со средой
        if hasattr(self, 'agents') and self.agents:
            for agent in self.agents:
                agent.thoughts.data = torch.randn(agent.latent_size).to(device)  # Опционально: Сброс состояния

        if self.rendering:
            self.render()

        return self.get_observations()

    def generate_maze(self, width, height):
        # Recursive Backtracker Maze Generator
        maze = np.ones((width, height), dtype=int)
        stack = []

        # Starting point (ensure it's odd to guarantee carvability)
        start_x, start_y = 1, 1 
        maze[start_x][start_y] = 0
        stack.append((start_x, start_y))

        while stack:
            current = stack[-1]
            x, y = current
            directions = [(2, 0), (-2, 0), (0, 2), (0, -2)]
            random.shuffle(directions)
            carved = False
            for dx, dy in directions:
                nx, ny = x + dx, y + dy
                if 0 < nx < width -1 and 0 < ny < height -1 and maze[nx][ny] == 1:
                    maze[nx][ny] = 0
                    maze[x + dx // 2][y + dy // 2] = 0
                    stack.append((nx, ny))
                    carved = True
                    break
            if not carved:
                stack.pop()
        return maze

    def print_maze(self):
        """Вывод лабиринта в консоль для отладки."""
        maze_str = ""
        for row in self.maze:
            maze_str += ''.join(['██' if cell == 1 else '  ' for cell in row]) + '\n'
        print(maze_str)

    def get_observations(self):
        observations = []
        for pos in self.agent_positions:
            obs = self.get_observation(pos)
            observations.append(obs)
        return observations

    def get_observation(self, position):
        # Create a grid centered at the agent's position with size visibility_radius*2+1
        x, y = position
        vr = self.visibility_radius
        obs = np.ones((2*vr+1, 2*vr+1), dtype=int)  # 0 - free, 1 - wall, 2 - agent, 3 - goal

        for i in range(-vr, vr+1):
            for j in range(-vr, vr+1):
                nx, ny = x + i, y + j
                if 0 <= nx < self.maze_size[0] and 0 <= ny < self.maze_size[1]:
                    obs[i+vr][j+vr] = self.maze[nx][ny]
                    # Check for agents
                    for idx, agent_pos in enumerate(self.agent_positions):
                        if agent_pos == (nx, ny):
                            obs[i+vr][j+vr] = 2
                    # Check for goal
                    if (nx, ny) == self.goal_position:
                        obs[i+vr][j+vr] = 3
        return obs.flatten()  # Flatten the observation for the neural network

    def step(self, actions, incoming_waveforms):
        rewards = [0 for _ in range(self.num_agents)]
        infos = {}

        # Handle incoming waveforms
        for idx, waveform in enumerate(incoming_waveforms):
            if waveform is not None:
                # Декодируем и интегрируем полученную волновую форму в размышления агента
                self.agents[idx].receive_waveform(waveform)
                rewards[idx] += 1  # Награда за успешное получение сообщения
                print(f"Agent {idx} received and integrated a message.")  # Debugging

        # Move agents
        new_positions = self.agent_positions.copy()
        for idx, action in enumerate(actions):
            if action < 0 or action > 5:
                continue  # Invalid action
            current_pos = self.agent_positions[idx]
            new_pos = list(current_pos)
            if action == 0:  # Up
                new_pos[0] -= 1
            elif action == 1:  # Down
                new_pos[0] += 1
            elif action == 2:  # Left
                new_pos[1] -= 1
            elif action == 3:  # Right
                new_pos[1] += 1
            elif action == 4:  # Stay
                pass
            elif action == 5:  # Send Message
                pass  # Сообщение уже обработано выше

            # Boundary and wall checks
            if 0 <= new_pos[0] < self.maze_size[0] and 0 <= new_pos[1] < self.maze_size[1]:
                if self.maze[new_pos[0]][new_pos[1]] == 0 and tuple(new_pos) not in self.agent_positions:
                    new_positions[idx] = tuple(new_pos)
                    rewards[idx] += 0.1  # Небольшая награда за движение
                    print(f"Agent {idx} moved to {new_positions[idx]}")  # Debugging: Print movement
                else:
                    print(f"Agent {idx} hit a wall or another agent at {tuple(new_pos)}")  # Debugging
            else:
                print(f"Agent {idx} tried to move out of bounds to {tuple(new_pos)}")  # Debugging

            # Check if reached the goal
            if new_positions[idx] == self.goal_position:
                rewards[idx] += 10  # Reward for reaching the goal
                self.done = True
                print(f"Agent {idx} reached the goal!")  # Debugging

        self.agent_positions = new_positions.copy()

        # Additional reward for agents moving closer to each other
        if self.num_agents > 1:
            total_distance = sum([
                abs(self.agent_positions[i][0] - self.agent_positions[j][0]) + 
                abs(self.agent_positions[i][1] - self.agent_positions[j][1])
                for i in range(self.num_agents) for j in range(i+1, self.num_agents)
            ])
            num_pairs = self.num_agents * (self.num_agents -1) / 2
            avg_distance = total_distance / num_pairs if num_pairs > 0 else total_distance
            # Reward based on inverse of average distance
            for i in range(self.num_agents):
                rewards[i] += (1 / (avg_distance + 1))
            print(f"Average agents' distance: {avg_distance:.2f}, Rewards updated: {rewards}")  # Debugging
        else:
            avg_distance = None  # Only one agent

        # Prepare observations
        observations = self.get_observations()
        dones = [self.done for _ in range(self.num_agents)]
        info = {"distance": avg_distance}

        if self.rendering:
            self.render()

        return observations, rewards, dones, info

    def render(self):
        # Handle pygame events
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                exit()

        # Fill background
        self.screen.fill(self.FREE_COLOR)

        # Draw maze
        for i in range(self.maze_size[0]):
            for j in range(self.maze_size[1]):
                rect = pygame.Rect(j * self.CELL_SIZE, i * self.CELL_SIZE, self.CELL_SIZE, self.CELL_SIZE)
                if self.maze[i][j] == 1:
                    pygame.draw.rect(self.screen, self.WALL_COLOR, rect)
                else:
                    pygame.draw.rect(self.screen, self.FREE_COLOR, rect)
                pygame.draw.rect(self.screen, (200,200,200), rect, 1)  # Cell border

        # Draw goal
        goal_x, goal_y = self.goal_position
        goal_rect = pygame.Rect(goal_y * self.CELL_SIZE, goal_x * self.CELL_SIZE, self.CELL_SIZE, self.CELL_SIZE)
        pygame.draw.rect(self.screen, self.GOAL_COLOR, goal_rect)

        # Draw agents
        for idx, pos in enumerate(self.agent_positions):
            agent_color = self.AGENT_COLORS[idx % len(self.AGENT_COLORS)]
            agent_x, agent_y = pos
            center = (agent_y * self.CELL_SIZE + self.CELL_SIZE // 2, agent_x * self.CELL_SIZE + self.CELL_SIZE // 2)
            radius = self.CELL_SIZE // 3
            pygame.draw.circle(self.screen, agent_color, center, radius)

            # Optional: Draw communication lines
            # Здесь можно добавить визуализацию коммуникаций при необходимости

        # Draw Debug Text (Optional)
        font = pygame.font.SysFont(None, 24)
        debug_text = f"Agents: {self.agent_positions}, Goal: {self.goal_position}"
        text_surface = font.render(debug_text, True, (255, 0, 0))
        self.screen.blit(text_surface, (10, 10))

        pygame.display.flip()
        self.clock.tick(10)  # Limit to 10 FPS to reduce CPU usage
    
# ----------------------------
# 3. Agent Definition
# ----------------------------

class Agent(nn.Module):
    def __init__(self, observation_space, action_space, latent_size=128):
        super(Agent, self).__init__()
        self.observation_space = observation_space
        self.action_space = action_space
        self.latent_size = latent_size

        # Внутреннее латентное состояние (размышления)
        self.thoughts = nn.Parameter(torch.randn(latent_size))  # Инициализируется случайно

        # Определение сети для обработки наблюдений
        self.obs_net = nn.Sequential(
            nn.Linear(observation_space, 128),
            nn.ReLU(),
            nn.Linear(128, 128),
            nn.ReLU()
        )

        # Политическая сеть
        self.policy_net = nn.Sequential(
            nn.Linear(128 + latent_size, 128),
            nn.ReLU(),
            nn.Linear(128, action_space)
        )

        # Сеть оценки ценности
        self.value_net = nn.Sequential(
            nn.Linear(128 + latent_size, 128),
            nn.ReLU(),
            nn.Linear(128, 1)
        )

    def forward(self, obs, incoming_thoughts=None):
        # Обработка наблюдения
        obs_encoded = self.obs_net(obs)

        # Интеграция входящих размышлений, если есть
        if incoming_thoughts is not None:
            # Простая интеграция: усреднение существующих размышлений с входящими
            self.thoughts.data = (self.thoughts.data + incoming_thoughts) / 2

        # Объединение кодированного наблюдения с внутренними размышлениями
        thoughts_expanded = self.thoughts.unsqueeze(0).expand(obs_encoded.size(0), -1)  # Shape: (batch_size, latent_size)
        combined = torch.cat([obs_encoded, thoughts_expanded], dim=1)  # Shape: (batch_size, 128 + latent_size)

        # Вычисление политики и ценности
        policy = self.policy_net(combined)
        value = self.value_net(combined)
        return policy, value

    def select_action(self, obs, epsilon=0.1):
        self.eval()
        with torch.no_grad():
            obs = obs.unsqueeze(0)  # Shape: (1, observation_space)
            policy, value = self.forward(obs)
            if random.random() < epsilon:
                action = random.randint(0, self.action_space - 1)
            else:
                action = torch.argmax(policy, dim=1).item()
        self.train()
        return action

    def send_thoughts_waveform(self):
        with torch.no_grad():
            latent_vector = self.thoughts.clone().detach()
            inputs = audio_processor(raw_audio=latent_vector.cpu(), sampling_rate=audio_processor.sampling_rate, return_tensors="pt").to(device)
            encoder_outputs = audio_model.encode(inputs["input_values"], inputs["padding_mask"])
            padding_mask = inputs["padding_mask"]
        return encoder_outputs, padding_mask

    def receive_waveform(self, latent_vector):
        with torch.no_grad():
            inputs = audio_processor(raw_audio=latent_vector, sampling_rate=audio_processor.sampling_rate, return_tensors="pt")
            incoming_latent = audio_model.encode(inputs["input_values"], inputs["padding_mask"])
            incoming_latent_tensor = torch.tensor(incoming_latent, dtype=torch.float32).to(device)
            # Интегрируем входящие размышления
            self.thoughts.data = (self.thoughts.data + incoming_latent_tensor.squeeze()) / 2
    
# ----------------------------
# 4. Replay Buffer
# ----------------------------

class ReplayBuffer:
    def __init__(self, capacity=10000):
        self.buffer = deque(maxlen=capacity)

    def push(self, state, action, reward, next_state, done, message_waveform):
        self.buffer.append((state, action, reward, next_state, done, message_waveform))

    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size)
        state, action, reward, next_state, done, message_waveform = zip(*batch)
        return state, action, reward, next_state, done, message_waveform

    def __len__(self):
        return len(self.buffer)
    
# ----------------------------
# 5. Agent Trainers
# ----------------------------

class AgentTrainer:
    def __init__(self, agents, optimizers, replay_buffers, gamma=0.99, batch_size=64):
        self.agents = agents
        self.optimizers = optimizers
        self.replay_buffers = replay_buffers
        self.gamma = gamma
        self.batch_size = batch_size

    def compute_loss(self, agent_idx):
        agent = self.agents[agent_idx]
        optimizer = self.optimizers[agent_idx]
        replay_buffer = self.replay_buffers[agent_idx]

        if len(replay_buffer) < self.batch_size:
            return

        state, action, reward, next_state, done, message_waveform = replay_buffer.sample(self.batch_size)

        state = torch.FloatTensor(state).to(device)
        next_state = torch.FloatTensor(next_state).to(device)
        action = torch.LongTensor(action).to(device)
        reward = torch.FloatTensor(reward).to(device)
        done = torch.FloatTensor(done).to(device)

        # Current policy and value
        policy, value = agent(state)
        action_log_probs = F.softmax(policy, dim=-1)
        selected_actions = action.unsqueeze(1)
        log_probs = torch.log(torch.gather(action_log_probs, 1, selected_actions).squeeze())

        # Compute target values
        with torch.no_grad():
            next_policy, next_value = agent(next_state)
            target_value = reward + self.gamma * next_value.squeeze() * (1 - done)

        # Compute advantage
        advantage = target_value - value.squeeze()

        # Policy loss
        policy_loss = -torch.mean(log_probs * advantage)

        # Value loss
        value_loss = F.mse_loss(value.squeeze(), target_value)

        # Total loss
        loss = policy_loss + value_loss

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # Print loss values
        print(f"Agent {agent_idx} - Policy Loss: {policy_loss.item():.4f}, Value Loss: {value_loss.item():.4f}")

    def train(self):
        for idx in range(len(self.agents)):
            self.compute_loss(idx)

# ----------------------------
# 6. Training Loop
# ----------------------------

MAZE_SIZE = (21, 21)
NUM_AGENTS = 2
VISIBILITY_RADIUS = 3
NUM_EPISODES = 1000
MAX_STEPS = 100
RENDER = True  # Установите True для визуализации

# Initialize environment
env = MazeEnvironment(maze_size=MAZE_SIZE, num_agents=NUM_AGENTS, visibility_radius=VISIBILITY_RADIUS, render=RENDER)

# Automatically compute observation_space
observation_space = (2 * env.visibility_radius + 1) ** 2  # For visibility_radius=3, observation_space=49

# Initialize agents
agents: list[Agent] = []
optimizers = []
replay_buffers = []
for _ in range(env.num_agents):
    agent = Agent(observation_space=observation_space, action_space=6, latent_size=128).to(device)
    agents.append(agent)
    optimizer = optim.Adam(agent.parameters(), lr=1e-3)
    optimizers.append(optimizer)
    replay_buffers.append(ReplayBuffer(capacity=10000))

# Pass agent list to environment for access in step
env.agents = agents  # Добавляем атрибут agents в класс среды

# Initialize trainer
trainer = AgentTrainer(agents, optimizers, replay_buffers, gamma=0.99, batch_size=64)

# Training statistics
success_count = 0
reward_history = []

for episode in range(1, NUM_EPISODES+1):
    try:
        observations = env.reset()
    except Exception as e:
        print(f"Error during environment reset: {e}")
        exit(1)
    total_rewards = [0 for _ in range(env.num_agents)]
    for step in range(MAX_STEPS):
        actions = []
        waveforms_to_send = []

        for idx, agent in enumerate(agents):
            obs = torch.FloatTensor(observations[idx]).to(device)
            action = agent.select_action(obs, epsilon=0.1)
            actions.append(action)

            if action == 5:  # Send Message
                # Генерируем токены из внутренних размышлений
                encoded_input, padding_mask = agent.send_thoughts_waveform()
                # Преобразуем токены обратно в волновую форму
                waveform: torch.Tensor = audio_model.decode(encoded_input.audio_codes, encoded_input.audio_scales, padding_mask)[0]
                waveform = waveform.to('cpu')
                waveforms_to_send.append(waveform.detach().numpy())
            else:
                waveforms_to_send.append(None)

        # Distribute the sent waveforms to other agents
        incoming_waveforms = [[] for _ in range(env.num_agents)]
        for sender_idx, waveform in enumerate(waveforms_to_send):
            if waveform is not None:
                for receiver_idx in range(env.num_agents):
                    if receiver_idx != sender_idx:
                        incoming_waveforms[receiver_idx].append(waveform)

        # For each agent, average incoming waveforms if multiple
        final_incoming_waveforms = []
        for receiver_idx in range(env.num_agents):
            if incoming_waveforms[receiver_idx]:
                combined_waveform = np.mean(incoming_waveforms[receiver_idx], axis=0)
                final_incoming_waveforms.append(combined_waveform)
            else:
                final_incoming_waveforms.append(None)

        # Step the environment with actions and incoming_waveforms
        next_observations, rewards, dones, info = env.step(actions, final_incoming_waveforms)

        # Store transitions in replay buffer
        for idx in range(env.num_agents):
            replay_buffers[idx].push(
                state=observations[idx],
                action=actions[idx],
                reward=rewards[idx],
                next_state=next_observations[idx],
                done=dones[idx],
                message_waveform=final_incoming_waveforms[idx]
            )
            total_rewards[idx] += rewards[idx]

        observations = next_observations

        if any(dones):
            break

        # If rendering, delay to make visualization visible
        if RENDER:
            env.clock.tick(10)  # Keep the rendering at 10 FPS

    # Training step
    trainer.train()

    # Statistics
    episode_reward = sum(total_rewards)
    reward_history.append(episode_reward)
    if sum([1 for r in total_rewards if r >=10 ]) >0:
        success_count +=1

    if episode % 100 == 0:
        avg_reward = np.mean(reward_history[-100:])
        success_rate = success_count / 100
        print(f"Episode {episode}, Average Reward: {avg_reward:.2f}, Success Rate: {success_rate*100:.2f}%")
        success_count = 0

# Plot reward history
plt.plot(reward_history)
plt.xlabel('Episode')
plt.ylabel('Total Reward')
plt.title('Training Reward History')
plt.show()

if RENDER:
    pygame.quit()
```
Rewrite this code to use the "VocalizationNetwork" - designed by you nn, that will encode agent's latent to waveform and decode back.