Traceback (most recent call last):
File "C:\Users\hp\Downloads\girs.py", line 610, in
train(epochs=350, render=True) # Change to False if you want to disable rendering
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\hp\Downloads\girs.py", line 563, in train
agent = Agent(render=render)
^^^^^^^^^^^^^^^^^^^^
File "C:\Users\hp\Downloads\girs.py", line 336, in init
self.load_model()
File "C:\Users\hp\Downloads\girs.py", line 341, in load_model
self.model.load_state_dict(torch.load('model.pth', map_location=device))
File "C:\Users\hp\AppData\Local\Programs\Python\Python312\Lib\site-packages\torch\nn\modules\module.py", line 2153, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for DuelingDQN:
size mismatch for fc1.weight: copying a param with shape torch.Size([256, 11]) from checkpoint, the shape in current model is torch.Size([2120, 11]).
size mismatch for fc1.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([2120]).
size mismatch for fc_adv.weight: copying a param with shape torch.Size([256, 256]) from checkpoint, the shape in current model is torch.Size([2120, 2120]).
size mismatch for fc_adv.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([2120]).
size mismatch for fc_adv_out.weight: copying a param with shape torch.Size([3, 256]) from checkpoint, the shape in current model is torch.Size([3, 2120]).
size mismatch for fc_val.weight: copying a param with shape torch.Size([256, 256]) from checkpoint, the shape in current model is torch.Size([2120, 2120]).
size mismatch for fc_val.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([2120]).
size mismatch for fc_val_out.weight: copying a param with shape torch.Size([1, 256]) from checkpoint, the shape in current model is torch.Size([1, 2120]).

```
import os
import pygame
import sys
import random
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from collections import deque, namedtuple
import heapq  # For A* implementation

# -------------------- Game Constants --------------------
WINDOW_WIDTH = 600
WINDOW_HEIGHT = 600
BLOCK_SIZE = 20
SPEED = 30

# Colors
WHITE = (255, 255, 255)
GREY = (220, 220, 220)
BLACK = (0, 0, 0)
RED = (200, 0, 0)
GREEN = (50, 168, 82)
BLUE = (0, 0, 255)

# Initialize Pygame
pygame.init()
pygame.display.set_caption('Advanced Snake AI v5')
font = pygame.font.SysFont('Arial', 25)
screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
clock = pygame.time.Clock()

# Check for CUDA availability
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f'Using device: {device}')

# -------------------- Prioritized Replay Memory with Prioritized Replay --------------------
class PrioritizedReplayMemory:
    def __init__(self, capacity, alpha=0.6):
        self.capacity = capacity
        self.alpha = alpha
        self.memory = []
        self.pos = 0
        self.priorities = np.zeros((capacity,), dtype=np.float32)
        self.Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done'))

    def push(self, *args):
        max_priority = self.priorities.max() if len(self.memory) > 0 else 1.0
        max_priority = max(max_priority, 1e-5)  # Ensure max_priority is at least 1e-5
        if len(self.memory) < self.capacity:
            self.memory.append(self.Transition(*args))
        else:
            self.memory[self.pos] = self.Transition(*args)
        self.priorities[self.pos] = max_priority
        self.pos = (self.pos + 1) % self.capacity

    def sample(self, batch_size, beta=0.4):
        if len(self.memory) == self.capacity:
            priorities = self.priorities
        else:
            priorities = self.priorities[:self.pos]

        # Avoid zero priorities
        priorities += 1e-5
        probabilities = priorities ** self.alpha
        probabilities /= probabilities.sum()

        indices = np.random.choice(len(self.memory), batch_size, p=probabilities)
        samples = [self.memory[idx] for idx in indices]

        total = len(self.memory)
        weights = (total * probabilities[indices]) ** (-beta)
        weights /= weights.max()
        weights = torch.tensor(weights, dtype=torch.float32, device=device)

        batch = self.Transition(*zip(*samples))
        return batch, indices, weights

    def update_priorities(self, batch_indices, batch_priorities):
        for idx, priority in zip(batch_indices, batch_priorities):
            self.priorities[idx] = max(priority.item(), 1e-5)  # Ensure priorities are at least 1e-5

    def __len__(self):
        return len(self.memory)

# -------------------- Dueling DQN Model --------------------
class DuelingDQN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(DuelingDQN, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.relu = nn.ReLU()

        # Advantage Stream
        self.fc_adv = nn.Linear(hidden_size, hidden_size)
        self.fc_adv_out = nn.Linear(hidden_size, output_size)

        # Value Stream
        self.fc_val = nn.Linear(hidden_size, hidden_size)
        self.fc_val_out = nn.Linear(hidden_size, 1)

    def forward(self, state):
        x = self.relu(self.fc1(state))

        adv = self.relu(self.fc_adv(x))
        adv = self.fc_adv_out(adv)

        val = self.relu(self.fc_val(x))
        val = self.fc_val_out(val)

        # Combine streams
        q_vals = val + adv - adv.mean(dim=1, keepdim=True)
        return q_vals

# -------------------- A* Pathfinding Algorithm Implementation --------------------
class AStar:
    def __init__(self, game):
        self.game = game
        self.grid_width = WINDOW_WIDTH // BLOCK_SIZE
        self.grid_height = WINDOW_HEIGHT // BLOCK_SIZE

    def heuristic(self, a, b):
        # Use Manhattan distance as heuristic
        return abs(a[0] - b[0]) + abs(a[1] - b[1])

    def get_neighbors(self, node):
        neighbors = []
        directions = [(-1, 0), (1, 0), (0, -1), (0, 1)]  # Left, Right, Up, Down
        for dx, dy in directions:
            x2 = node[0] + dx
            y2 = node[1] + dy
            if 0 <= x2 < self.grid_width and 0 <= y2 < self.grid_height:
                neighbors.append((x2, y2))
        return neighbors

    def find_path(self):
        start = (self.game.head_x // BLOCK_SIZE, self.game.head_y // BLOCK_SIZE)
        goal = (self.game.food_x // BLOCK_SIZE, self.game.food_y // BLOCK_SIZE)

        snake = set((x // BLOCK_SIZE, y // BLOCK_SIZE) for x, y in self.game.snake_body)

        frontier = []
        heapq.heappush(frontier, (0, start))
        came_from = {}
        cost_so_far = {}
        came_from[start] = None
        cost_so_far[start] = 0

        while frontier:
            _, current = heapq.heappop(frontier)

            if current == goal:
                break

            for neighbor in self.get_neighbors(current):
                if neighbor in snake and neighbor != goal:
                    continue  # Avoid snake's body except the food position

                new_cost = cost_so_far[current] + 1  # All moves have the same cost
                if neighbor not in cost_so_far or new_cost < cost_so_far[neighbor]:
                    cost_so_far[neighbor] = new_cost
                    priority = new_cost + self.heuristic(goal, neighbor)
                    heapq.heappush(frontier, (priority, neighbor))
                    came_from[neighbor] = current

        # Reconstruct path
        if goal not in came_from:
            return []  # No path found

        current = goal
        path = []
        while current != start:
            path.append(current)
            current = came_from[current]
        path.reverse()
        return path  # List of tuples

# -------------------- Snake Game Class --------------------
class SnakeGameAI:
    def __init__(self):
        self.reset()

    def reset(self):
        self.direction = random.choice(['UP', 'DOWN', 'LEFT', 'RIGHT'])
        self.head_x = WINDOW_WIDTH // 2
        self.head_y = WINDOW_HEIGHT // 2
        self.snake_body = [(self.head_x, self.head_y)]
        self.length = 1

        self.place_food()
        self.frame_iteration = 0
        self.score = 0
        self.reward = 0
        self.game_over = False

    def play_step(self, action, render=True, path=None):
        self.frame_iteration += 1

        # Collect events (this is necessary to keep the window responsive)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

        # Compute distance before move
        current_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)

        # Move
        self.move(action)
        self.snake_body.insert(0, (self.head_x, self.head_y))

        # Compute distance after move
        new_distance = abs(self.head_x - self.food_x) + abs(self.head_y - self.food_y)

        # Check for collision
        self.reward = 0
        self.game_over = False
        if self.is_collision() or self.frame_iteration > 100 * self.length:
            self.game_over = True
            self.reward = -10
            return self.reward, self.game_over, self.score

        # Check for food collision
        if self.head_x == self.food_x and self.head_y == self.food_y:
            self.score += 1
            self.reward = 10
            self.length += 1
            self.place_food()
        else:
            self.snake_body.pop()
            # Encourage moving towards food
            if new_distance < current_distance:
                self.reward = 0.1
            else:
                self.reward = -0.1

        # Draw game
        if render:
            self.update_ui()
            clock.tick(SPEED)

        return self.reward, self.game_over, self.score

    def is_collision(self, pt=None):
        if pt is None:
            pt = (self.head_x, self.head_y)
        # Hits boundary
        if pt[0] < 0 or pt[0] >= WINDOW_WIDTH or pt[1] < 0 or pt[1] >= WINDOW_HEIGHT:
            return True
        # Hits itself
        if pt in self.snake_body[1:]:
            return True
        return False

    def move(self, action):
        # [straight, right, left]
        clock_wise = ['RIGHT', 'DOWN', 'LEFT', 'UP']
        idx = clock_wise.index(self.direction)

        if action == 0:
            new_dir = clock_wise[idx]  # No change
        elif action == 1:
            next_idx = (idx + 1) % 4
            new_dir = clock_wise[next_idx]  # Right turn
        elif action == 2:
            next_idx = (idx - 1) % 4
            new_dir = clock_wise[next_idx]  # Left turn
        else:
            new_dir = self.direction  # Default to current direction

        self.direction = new_dir

        # Move the head
        if self.direction == 'RIGHT':
            self.head_x += BLOCK_SIZE
        elif self.direction == 'LEFT':
            self.head_x -= BLOCK_SIZE
        elif self.direction == 'UP':
            self.head_y -= BLOCK_SIZE
        elif self.direction == 'DOWN':
            self.head_y += BLOCK_SIZE

    def place_food(self):
        while True:
            self.food_x = random.randint(0, (WINDOW_WIDTH - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
            self.food_y = random.randint(0, (WINDOW_HEIGHT - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
            if (self.food_x, self.food_y) not in self.snake_body:
                break

    def update_ui(self):
        screen.fill(BLACK)
        for block in self.snake_body:
            pygame.draw.rect(screen, GREEN, pygame.Rect(block[0], block[1], BLOCK_SIZE, BLOCK_SIZE))
        pygame.draw.rect(screen, RED, pygame.Rect(self.food_x, self.food_y, BLOCK_SIZE, BLOCK_SIZE))
        text = font.render(f'Score: {self.score}', True, WHITE)
        screen.blit(text, [0, 0])
        pygame.display.flip()

    def get_grid(self):
        """Returns the current grid with obstacles."""
        grid = np.zeros((self.grid_height, self.grid_width), dtype=int)
        # Mark the snake's body as obstacles
        for x, y in self.snake_body[1:]:
            grid[y // BLOCK_SIZE][x // BLOCK_SIZE] = 1  # 1 represents obstacle
        return grid

    @property
    def grid_width(self):
        return WINDOW_WIDTH // BLOCK_SIZE

    @property
    def grid_height(self):
        return WINDOW_HEIGHT // BLOCK_SIZE

# -------------------- Agent Class with Advanced Strategies --------------------
class Agent:
    def __init__(self, render=False):
        self.n_games = 0
        self.epsilon = 1.0  # Exploration rate
        self.eps_min = 0.01
        self.eps_decay = 0.995
        self.gamma = 0.9  # Discount rate
        self.lr = 0.0001  # Lowered learning rate for larger model
        self.memory = PrioritizedReplayMemory(100_000)
        self.batch_size = 512
        self.model = DuelingDQN(11, 2120, 3).to(device)  # Updated hidden_size to 2120 for ~9M parameters
        self.target_model = DuelingDQN(11, 2120, 3).to(device)
        self.update_target_network()
        self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
        self.criterion = nn.SmoothL1Loss()
        self.update_frequency = 1000
        self.learn_step_counter = 0
        self.beta_start = 0.4
        self.beta_frames = 100_000
        self.beta = self.beta_start
        self.render = render
        self.load_model()
        self.a_star = None  # Will be initialized per game

    def load_model(self):
        if os.path.exists('model.pth'):
            self.model.load_state_dict(torch.load('model.pth', map_location=device))
            self.target_model.load_state_dict(torch.load('model.pth', map_location=device))
            print("Model loaded from model.pth")

    def save_model(self):
        torch.save(self.model.state_dict(), 'model.pth')
        print("Model saved to model.pth")

    def update_target_network(self):
        self.target_model.load_state_dict(self.model.state_dict())

    def get_state(self, game):
        head = game.snake_body[0]
        point_l = (head[0] - BLOCK_SIZE, head[1])
        point_r = (head[0] + BLOCK_SIZE, head[1])
        point_u = (head[0], head[1] - BLOCK_SIZE)
        point_d = (head[0], head[1] + BLOCK_SIZE)

        dir_l = game.direction == 'LEFT'
        dir_r = game.direction == 'RIGHT'
        dir_u = game.direction == 'UP'
        dir_d = game.direction == 'DOWN'

        state = [
            # Danger straight
            (dir_r and game.is_collision(point_r)) or
            (dir_l and game.is_collision(point_l)) or
            (dir_u and game.is_collision(point_u)) or
            (dir_d and game.is_collision(point_d)),

            # Danger right
            (dir_u and game.is_collision(point_r)) or
            (dir_d and game.is_collision(point_l)) or
            (dir_l and game.is_collision(point_u)) or
            (dir_r and game.is_collision(point_d)),

            # Danger left
            (dir_d and game.is_collision(point_r)) or
            (dir_u and game.is_collision(point_l)) or
            (dir_r and game.is_collision(point_u)) or
            (dir_l and game.is_collision(point_d)),

            # Move direction
            dir_l,
            dir_r,
            dir_u,
            dir_d,

            # Food location
            game.food_x < game.head_x,  # food left
            game.food_x > game.head_x,  # food right
            game.food_y < game.head_y,  # food up
            game.food_y > game.head_y,  # food down
        ]
        return np.array(state, dtype=int)

    def remember(self, state, action, reward, next_state, done):
        # Store transitions as is (action is scalar)
        self.memory.push(state, action, reward, next_state, done)

    def train_long_memory(self):
        if len(self.memory) < self.batch_size:
            return  # Not enough samples
        transitions, indices, weights = self.memory.sample(self.batch_size, beta=self.beta)
        self.beta = min(1.0, self.beta + (1.0 - self.beta_start) / self.beta_frames)

        # Convert batch arrays to tensors
        states = torch.tensor(transitions.state, dtype=torch.float32, device=device)
        actions = torch.tensor(transitions.action, dtype=torch.long, device=device)
        rewards = torch.tensor(transitions.reward, dtype=torch.float32, device=device)
        next_states = torch.tensor(transitions.next_state, dtype=torch.float32, device=device)
        dones = torch.tensor(transitions.done, dtype=torch.bool, device=device)

        self.train_step(states, actions, rewards, next_states, dones, indices, weights)

    def train_step(self, states, actions, rewards, next_states, dones, indices, weights):
        # Compute current Q values
        current_q_values = self.model(states).gather(1, actions.unsqueeze(1)).squeeze(1)

        with torch.no_grad():
            # Double DQN: select the best action using the online network
            next_actions = self.model(next_states).argmax(1)
            # Evaluate the next Q values using the target network
            next_q_values = self.target_model(next_states).gather(1, next_actions.unsqueeze(1)).squeeze(1)
            # Zero out the Q values for terminal states
            next_q_values[dones] = 0.0

        # Expected Q values
        expected_q_values = rewards + self.gamma * next_q_values

        # Compute TD error
        td_errors = expected_q_values - current_q_values
        loss = (weights * td_errors.pow(2)).mean()

        # Backpropagation
        self.optimizer.zero_grad()
        loss.backward()
        # Gradient clipping to prevent exploding gradients
        nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0)
        self.optimizer.step()

        # Update priorities in the replay memory
        priorities = td_errors.abs().detach() + 1e-5
        self.memory.update_priorities(indices, priorities)

        # Update target network periodically
        self.learn_step_counter += 1
        if self.learn_step_counter % self.update_frequency == 0:
            self.update_target_network()
            # Save model periodically
            self.save_model()

    def get_action(self, state, game):
        self.epsilon = max(self.eps_min, self.epsilon * self.eps_decay)
        final_move = 0  # Default to 'straight'

        # Initialize A* pathfinder if not already
        if self.a_star is None:
            self.a_star = AStar(game)

        # Attempt to find path using A*
        path = self.a_star.find_path()

        if path:
            # Determine the next move from the path
            next_cell = path[0]
            current = (game.head_x // BLOCK_SIZE, game.head_y // BLOCK_SIZE)
            direction = (next_cell[0] - current[0], next_cell[1] - current[1])

            if direction == (1, 0):
                desired_direction = 'RIGHT'
            elif direction == (-1, 0):
                desired_direction = 'LEFT'
            elif direction == (0, 1):
                desired_direction = 'DOWN'
            elif direction == (0, -1):
                desired_direction = 'UP'
            else:
                desired_direction = game.direction  # Default to current direction

            # Map desired direction to action
            desired_idx = ['RIGHT', 'DOWN', 'LEFT', 'UP'].index(desired_direction)
            current_idx = ['RIGHT', 'DOWN', 'LEFT', 'UP'].index(game.direction)
            turn = (desired_idx - current_idx) % 4

            if turn == 0:
                a_star_action = 0  # Straight
            elif turn == 1:
                a_star_action = 1  # Right
            elif turn == 3:
                a_star_action = 2  # Left
            else:
                a_star_action = 0  # Straight if turn == 2 (180 degrees, which is invalid)

            # Advanced Fallback: Ensure the A* action is safe
            point_action = self.get_action_point(a_star_action, game)
            if not game.is_collision_with_action(point_action):
                final_move = a_star_action
            else:
                # If A* action is not safe, use DQN
                final_move = self.get_dqn_action(state)
        else:
            # No path found, fallback to DQN action
            final_move = self.get_dqn_action(state)

        return final_move  # Return as scalar index

    def get_dqn_action(self, state):
        # Epsilon-greedy action selection
        if random.random() < self.epsilon:
            return random.randint(0, 2)  # 0: straight, 1: right, 2: left
        else:
            state_tensor = torch.tensor(state, dtype=torch.float32, device=device).unsqueeze(0)
            with torch.no_grad():
                q_values = self.model(state_tensor)
                # Survival Deviation: Adding noise to the action selection
                q_values += torch.randn_like(q_values) * 0.1
                final_move = torch.argmax(q_values).item()  # Choose the best action
            return final_move

    def get_action_point(self, action, game):
        # Predict the next point based on the action
        clock_wise = ['RIGHT', 'DOWN', 'LEFT', 'UP']
        idx = clock_wise.index(game.direction)

        if action == 0:
            new_dir = clock_wise[idx]  # No change
        elif action == 1:
            next_idx = (idx + 1) % 4
            new_dir = clock_wise[next_idx]  # Right turn
        elif action == 2:
            next_idx = (idx - 1) % 4
            new_dir = clock_wise[next_idx]  # Left turn
        else:
            new_dir = game.direction  # Default to current direction

        # Determine new head position
        if new_dir == 'RIGHT':
            return (game.head_x + BLOCK_SIZE, game.head_y)
        elif new_dir == 'LEFT':
            return (game.head_x - BLOCK_SIZE, game.head_y)
        elif new_dir == 'UP':
            return (game.head_x, game.head_y - BLOCK_SIZE)
        elif new_dir == 'DOWN':
            return (game.head_x, game.head_y + BLOCK_SIZE)
        else:
            return (game.head_x, game.head_y)

    def train_agent_with_a_star(self, game):
        # Initialize A* for the current game
        self.a_star = AStar(game)

# -------------------- Extend SnakeGameAI to include collision checking for potential actions --------------------
def is_collision_with_action(self, action_point):
    # Check if the action_point results in collision
    return self.is_collision(pt=action_point)

# Monkey patch the method to SnakeGameAI
setattr(SnakeGameAI, 'is_collision_with_action', is_collision_with_action)

# -------------------- Training Function --------------------
def train(epochs=350, render=False):
    agent = Agent(render=render)
    game = SnakeGameAI()
    best_score = 0
    try:
        for epoch in range(1, epochs + 1):
            state_old = agent.get_state(game)

            # Get action, incorporating A* guidance
            final_move = agent.get_action(state_old, game)

            # Perform action and get new state
            reward, done, score = game.play_step(final_move, render=render)
            state_new = agent.get_state(game)

            # Remember the transition
            agent.remember(state_old, final_move, reward, state_new, done)

            # Train long memory
            agent.train_long_memory()

            if done:
                # Reset game
                game.reset()
                agent.n_games += 1

                if score > best_score:
                    best_score = score
                    agent.save_model()

                print(f'Epoch {epoch}/{epochs} | Game {agent.n_games} | Score: {score} | Record: {best_score} | Epsilon: {agent.epsilon:.4f}')

                # Optionally, break early if desired performance is achieved
                # if best_score >= TARGET_SCORE:
                #     print("Target score achieved!")
                #     break

    except KeyboardInterrupt:
        print("Training interrupted by user.")
    finally:
        agent.save_model()

# -------------------- Extend SnakeGameAI with Collision Checking --------------------
# Already handled above with monkey patching

# -------------------- Main Execution --------------------
if __name__ == '__main__':
    # Set render to False for faster training without visualization
    train(epochs=350, render=True)  # Change to False if you want to disable rendering