FURTHER ENHANCE AND CONTINUE EXPANDING IT:

```
import pygame
import sys
import random
from collections import deque
import heapq
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

# Initialize Pygame
pygame.init()

# Game Constants
CELL_SIZE = 20
GRID_WIDTH = 30
GRID_HEIGHT = 20
WINDOW_WIDTH = CELL_SIZE * GRID_WIDTH + 200  # Extra width for UI
WINDOW_HEIGHT = CELL_SIZE * GRID_HEIGHT
FPS = 10

# Boost Constants
BOOST_DURATION = 50  # Number of frames boost is active
BOOST_COOLDOWN = 100  # Number of frames before boost can be activated again
BOOST_SPEED_MULTIPLIER = 2  # Multiplier for FPS when boost is active

# Q-Learning Constants
ACTIONS = [
    ("UP", False),
    ("DOWN", False),
    ("LEFT", False),
    ("RIGHT", False),
    ("UP", True),
    ("DOWN", True),
    ("LEFT", True),
    ("RIGHT", True)
]

ACTION_INDEX = {action: idx for idx, action in enumerate(ACTIONS)}
NUM_ACTIONS = len(ACTIONS)

# Reward Constants
REWARD_FOOD = 100
REWARD_STEP = 1
REWARD_BOOST_EFFECTIVE = 5
REWARD_CLOSE_DISTANCE = 2  # Bonus per unit distance reduced
PENALTY_FAR_DISTANCE = -1  # Penalty per unit distance increased
PENALTY_COLLISION = -100
PENALTY_BOOST_UNUSED = -2

# Neural Network Parameters
STATE_SIZE = 11  # Corrected to match the state representation
HIDDEN_SIZE = 150  # To achieve ~3K parameters
LEARNING_RATE = 0.001
BATCH_SIZE = 64
GAMMA = 0.9
EPSILON_START = 1.0
EPSILON_END = 0.01
EPSILON_DECAY = 0.995
MEMORY_SIZE = 10000
TARGET_UPDATE = 10  # Update target network every 10 episodes

# Colors
WHITE = (255, 255, 255)
GREY = (40, 40, 40)
BLACK = (0, 0, 0)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
BLUE = (0, 0, 255)
YELLOW = (255, 255, 0)

# Set up display
screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
pygame.display.set_caption("Advanced Snake AI with Deep Q-Learning and Boost")
clock = pygame.time.Clock()

# Initialize Font
pygame.font.init()
FONT_SIZE = 20
FONT = pygame.font.SysFont('Calibri', FONT_SIZE, bold=True)

def render_text(text, color, position):
    """
    Renders text on the screen at the given position.
    """
    text_surface = FONT.render(text, True, color)
    screen.blit(text_surface, position)

def render_text_with_background(text, color, position, bg_color=(0, 0, 0, 150)):
    """
    Renders text with a semi-transparent background.
    """
    text_surface = FONT.render(text, True, color)
    bg_surface = pygame.Surface((text_surface.get_width() + 10, text_surface.get_height() + 5), pygame.SRCALPHA)
    bg_surface.fill(bg_color)
    screen.blit(bg_surface, (position[0] - 5, position[1] - 2))
    screen.blit(text_surface, position)

class Snake:
    def __init__(self):
        self.positions = deque([(GRID_WIDTH // 2, GRID_HEIGHT // 2)])
        self.direction = (0, -1)  # Initially moving upwards
        self.grow = False
        self.hamiltonian = generate_hamiltonian_cycle()
        self.cycle_index = self.hamiltonian.index(self.head()) if self.head() in self.hamiltonian else 0
        # Boost attributes
        self.boost_active = False
        self.boost_duration = 0
        self.boost_cooldown = 0

    def head(self):
        return self.positions[0]

    def move(self, direction):
        if direction is not None:
            self.direction = direction
        new_x = self.head()[0] + self.direction[0]
        new_y = self.head()[1] + self.direction[1]
        new_position = (new_x, new_y)

        # Check wall collision
        if (0 <= new_x < GRID_WIDTH) and (0 <= new_y < GRID_HEIGHT) and new_position not in self.positions:
            self.positions.appendleft(new_position)
            if not self.grow:
                self.positions.pop()
            else:
                self.grow = False
            return True  # Successful move
        else:
            return False  # Collision occurred

    def grow_snake(self):
        self.grow = True

    def follow_hamiltonian(self):
        """
        Follows the next step in the Hamiltonian cycle.
        """
        self.cycle_index = (self.cycle_index + 1) % len(self.hamiltonian)
        next_cell = self.hamiltonian[self.cycle_index]
        direction = (next_cell[0] - self.head()[0], next_cell[1] - self.head()[1])
        return direction

    def activate_boost(self):
        if self.boost_cooldown == 0 and not self.boost_active:
            self.boost_active = True
            self.boost_duration = BOOST_DURATION
            self.boost_cooldown = BOOST_COOLDOWN

    def update_boost(self):
        if self.boost_active:
            self.boost_duration -= 1
            if self.boost_duration <= 0:
                self.boost_active = False
        if self.boost_cooldown > 0:
            self.boost_cooldown -= 1

class Food:
    def __init__(self, snake_positions):
        self.position = self.random_position(snake_positions)

    def random_position(self, snake_positions):
        while True:
            pos = (random.randint(0, GRID_WIDTH - 1), random.randint(0, GRID_HEIGHT - 1))
            if pos not in snake_positions:
                return pos

    def respawn(self, snake_positions):
        self.position = self.random_position(snake_positions)

def generate_hamiltonian_cycle():
    cycle = []
    for y in range(GRID_HEIGHT):
        row = list(range(GRID_WIDTH))
        if y % 2 == 1:
            row.reverse()
        for x in row:
            cycle.append((x, y))
    return cycle

def draw_grid():
    for x in range(0, GRID_WIDTH * CELL_SIZE, CELL_SIZE):
        pygame.draw.line(screen, GREY, (x, 0), (x, GRID_HEIGHT * CELL_SIZE))
    for y in range(0, GRID_HEIGHT * CELL_SIZE, CELL_SIZE):
        pygame.draw.line(screen, GREY, (0, y), (GRID_WIDTH * CELL_SIZE, y))

def draw_snake(snake):
    for pos in snake.positions:
        rect = pygame.Rect(pos[0] * CELL_SIZE, pos[1] * CELL_SIZE, CELL_SIZE, CELL_SIZE)
        pygame.draw.rect(screen, GREEN, rect)

def draw_food(food):
    rect = pygame.Rect(food.position[0] * CELL_SIZE, food.position[1] * CELL_SIZE, CELL_SIZE, CELL_SIZE)
    pygame.draw.rect(screen, RED, rect)

def draw_path(path):
    for pos in path:
        rect = pygame.Rect(pos[0] * CELL_SIZE, pos[1] * CELL_SIZE, CELL_SIZE, CELL_SIZE)
        pygame.draw.rect(screen, BLUE, rect, 2)  # Draw path with blue lines

def draw_hamiltonian_cycle(snake, cycle_length=50):
    start_index = snake.cycle_index
    end_index = (start_index + cycle_length) % len(snake.hamiltonian)
    if start_index < end_index:
        cycle_path = snake.hamiltonian[start_index:end_index]
    else:
        cycle_path = snake.hamiltonian[start_index:] + snake.hamiltonian[:end_index]

    for pos in cycle_path:
        rect = pygame.Rect(pos[0] * CELL_SIZE, pos[1] * CELL_SIZE, CELL_SIZE, CELL_SIZE)
        pygame.draw.rect(screen, WHITE, rect, 1)  # Draw cycle with white lines

def manhattan_distance(a, b):
    """Calculate Manhattan distance between two points a and b."""
    return abs(a[0] - b[0]) + abs(a[1] - b[1])

def get_state(snake, food, path_length=None):
    head_x, head_y = snake.head()
    food_x, food_y = food.position
    # Relative position of food
    food_direction = (
        1 if food_x > head_x else -1 if food_x < head_x else 0,
        1 if food_y > head_y else -1 if food_y < head_y else 0
    )
    # Immediate obstacles
    obstacles = [
        1 if (head_x + dx, head_y + dy) in snake.positions or not (0 <= head_x + dx < GRID_WIDTH and 0 <= head_y + dy < GRID_HEIGHT) else 0
        for dx, dy in [(-1,0), (1,0), (0,-1), (0,1)]
    ]
    # Boost availability
    boost_available = 1 if snake.boost_cooldown == 0 else 0
    # Current direction
    direction = snake.direction
    direction_binary = {
        (0, -1): [1, 0, 0, 0],  # Up
        (0, 1): [0, 1, 0, 0],   # Down
        (-1, 0): [0, 0, 1, 0],  # Left
        (1, 0): [0, 0, 0, 1]    # Right
    }
    state = direction_binary.get(direction, [0,0,0,0]) + list(food_direction) + obstacles + [boost_available]

    # Convert to tensor
    state_tensor = torch.tensor(state, dtype=torch.float32)
    return state_tensor

class ReplayMemory:
    def __init__(self, capacity):
        self.memory = deque(maxlen=capacity)

    def push(self, *args):
        """Save a transition"""
        self.memory.append(tuple(args))

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

class QNetwork(nn.Module):
    def __init__(self, input_size=11, hidden_size=150, output_size=NUM_ACTIONS):
        super(QNetwork, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)  # Input to hidden
        self.fc2 = nn.Linear(hidden_size, output_size)  # Hidden to output
        self._init_weights()

    def _init_weights(self):
        nn.init.xavier_uniform_(self.fc1.weight)
        nn.init.zeros_(self.fc1.bias)
        nn.init.xavier_uniform_(self.fc2.weight)
        nn.init.zeros_(self.fc2.bias)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class QLearningAgent:
    def __init__(self, actions, state_size=11, hidden_size=150, action_size=NUM_ACTIONS,
                 learning_rate=LEARNING_RATE, gamma=GAMMA, epsilon=EPSILON_START,
                 epsilon_decay=EPSILON_DECAY, epsilon_min=EPSILON_END,
                 memory_size=MEMORY_SIZE, batch_size=BATCH_SIZE, target_update=TARGET_UPDATE):
        self.actions = actions
        self.state_size = state_size
        self.action_size = action_size
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_decay = epsilon_decay
        self.epsilon_min = epsilon_min
        self.batch_size = batch_size
        self.target_update = target_update
        self.learn_step_counter = 0

        self.memory = ReplayMemory(memory_size)

        self.policy_net = QNetwork(input_size=state_size, hidden_size=hidden_size, output_size=action_size)
        self.target_net = QNetwork(input_size=state_size, hidden_size=hidden_size, output_size=action_size)
        self.target_net.load_state_dict(self.policy_net.state_dict())
        self.target_net.eval()

        self.optimizer = optim.Adam(self.policy_net.parameters(), lr=learning_rate)
        self.loss_fn = nn.MSELoss()

    def get_action(self, state, preferred_action=None):
        if preferred_action and random.random() > self.epsilon:
            # Follow the preferred action with exploitation
            action_idx = ACTION_INDEX.get(preferred_action, random.randint(0, self.action_size -1))
            return self.actions[action_idx]
        elif random.random() < self.epsilon:
            # Exploration
            return random.choice(self.actions)
        else:
            # Exploitation: choose the action with max Q-value
            with torch.no_grad():
                q_values = self.policy_net(state)
                action_idx = torch.argmax(q_values).item()
                return self.actions[action_idx]

    def remember(self, state, action, reward, next_state, done):
        action_idx = ACTION_INDEX.get(action, random.randint(0, self.action_size -1))
        self.memory.push(state, action_idx, reward, next_state, done)

    def replay(self):
        if len(self.memory) < self.batch_size:
            return

        transitions = self.memory.sample(self.batch_size)
        batch_state, batch_action, batch_reward, batch_next_state, batch_done = zip(*transitions)

        batch_state = torch.stack(batch_state)  # Shape: [batch_size, 11]
        batch_action = torch.tensor(batch_action, dtype=torch.int64).unsqueeze(1)  # Shape: [batch_size, 1]
        batch_reward = torch.tensor(batch_reward, dtype=torch.float32).unsqueeze(1)  # Shape: [batch_size, 1]
        batch_next_state = torch.stack(batch_next_state)  # Shape: [batch_size, 11]
        batch_done = torch.tensor(batch_done, dtype=torch.float32).unsqueeze(1)  # Shape: [batch_size, 1]

        # Current Q values
        current_q = self.policy_net(batch_state).gather(1, batch_action)

        # Next Q values
        with torch.no_grad():
            max_next_q = self.target_net(batch_next_state).max(1)[0].unsqueeze(1)
            target_q = batch_reward + (1 - batch_done) * self.gamma * max_next_q

        # Loss
        loss = self.loss_fn(current_q, target_q)

        # Optimize the model
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

    def update_epsilon(self):
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay

    def update_target_network(self):
        self.target_net.load_state_dict(self.policy_net.state_dict())

def a_star_search(start, goal, snake_positions):
    """
    Performs A* search from start to goal avoiding snake_positions.
    Returns the path as a list of positions from start to goal, inclusive.
    If no path found, returns an empty list.
    """
    def heuristic(a, b):
        return manhattan_distance(a, b)  # Manhattan distance

    open_set = []
    heapq.heappush(open_set, (heuristic(start, goal), 0, start, [start]))
    closed_set = set()

    while open_set:
        estimated_total, cost, current, path = heapq.heappop(open_set)

        if current == goal:
            return path

        if current in closed_set:
            continue
        closed_set.add(current)

        for dx, dy in [(-1,0), (1,0), (0,-1), (0,1)]:
            neighbor = (current[0] + dx, current[1] + dy)
            if (0 <= neighbor[0] < GRID_WIDTH and
                0 <= neighbor[1] < GRID_HEIGHT and
                neighbor not in snake_positions and
                neighbor not in closed_set):
                new_cost = cost + 1
                estimated = new_cost + heuristic(neighbor, goal)
                heapq.heappush(open_set, (estimated, new_cost, neighbor, path + [neighbor]))
    return []

def parse_action(action):
    direction, use_boost = action
    dir_map = {
        "UP": (0, -1),
        "DOWN": (0, 1),
        "LEFT": (-1, 0),
        "RIGHT": (1, 0)
    }
    return dir_map.get(direction, (0, -1)), use_boost

def draw_boost_indicator(snake):
    if snake.boost_active:
        head = snake.head()
        rect = pygame.Rect(head[0] * CELL_SIZE, head[1] * CELL_SIZE, CELL_SIZE, CELL_SIZE)
        pygame.draw.rect(screen, YELLOW, rect, 3)  # Highlight the head with a yellow border

def integrate_a_star(snake, food):
    """
    Integrates A* to determine the preferred action.
    Returns the preferred action tuple or None if no path is found.
    """
    path = a_star_search(snake.head(), food.position, set(snake.positions))
    if len(path) < 2:
        # No path found or already at food
        return None
    next_step = path[1]
    action_direction = (next_step[0] - snake.head()[0], next_step[1] - snake.head()[1])
    # Map direction to action
    dir_map = {
        (0, -1): "UP",
        (0, 1): "DOWN",
        (-1, 0): "LEFT",
        (1, 0): "RIGHT"
    }
    action_dir = dir_map.get(action_direction, "UP")
    # Prefer actions that are on the A* path
    preferred_action = (action_dir, False)  # Not using boost for path following
    return preferred_action

def main():
    snake = Snake()
    food = Food(snake.positions)
    running = True
    high_score = 0
    debug_mode = True  # Initialize debug mode as on
    agent = QLearningAgent(actions=ACTIONS)
    episode = 0

    state = get_state(snake, food)
    action = agent.get_action(state)
    direction, use_boost = parse_action(action)
    if use_boost:
        snake.activate_boost()

    # Track previous distance for reward calculation
    previous_distance = manhattan_distance(snake.head(), food.position)

    while running:
        # Adjust FPS based on boost status
        if snake.boost_active:
            clock.tick(FPS * BOOST_SPEED_MULTIPLIER)
        else:
            clock.tick(FPS)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                running = False
            elif event.type == pygame.KEYDOWN:
                if event.key == pygame.K_d:
                    debug_mode = not debug_mode  # Toggle debug mode

        # Integrate A* to get preferred action
        preferred_action = integrate_a_star(snake, food)

        # Decide whether to follow A* or Q-Learning based on epsilon
        action = agent.get_action(state, preferred_action)

        direction, use_boost = parse_action(action)
        if use_boost:
            snake.activate_boost()

        # Calculate current distance before action
        current_distance = manhattan_distance(snake.head(), food.position)

        # Take action
        moved = snake.move(direction)
        done = False
        reward = REWARD_STEP  # Default reward for surviving a step

        if not moved:
            # Collision occurred
            reward = PENALTY_COLLISION
            done = True

        # Check food collision
        if snake.head() == food.position:
            snake.grow_snake()
            food.respawn(snake.positions)
            reward = REWARD_FOOD

        # Update boost status
        snake.update_boost()

        # Calculate new distance after action
        new_distance = manhattan_distance(snake.head(), food.position)

        # Compute reward based on distance change
        distance_change = previous_distance - new_distance
        if distance_change > 0:
            # Moved closer to food
            reward += distance_change * REWARD_CLOSE_DISTANCE
        elif distance_change < 0:
            # Moved away from food
            reward += distance_change * PENALTY_FAR_DISTANCE

        # Compute reward based on boost usage
        if use_boost:
            if distance_change > 0:
                # Effective use of boost
                reward += REWARD_BOOST_EFFECTIVE
            else:
                # Ineffective use of boost
                reward += PENALTY_BOOST_UNUSED

        # Update previous distance
        previous_distance = new_distance

        # Observe new state
        next_state = get_state(snake, food)

        # Remember the experience
        agent.remember(state, action, reward, next_state, done)

        # Learn from the experience
        agent.replay()

        # Update state
        state = next_state

        # Update high score
        current_score = len(snake.positions) - 1
        if current_score > high_score:
            high_score = current_score

        # Rendering
        screen.fill(BLACK)
        draw_grid()
        draw_snake(snake)
        draw_food(food)

        # Optionally draw the A* path for visualization
        path = a_star_search(snake.head(), food.position, set(snake.positions))
        if path:
            draw_path(path)

        # Draw boost indicator
        draw_boost_indicator(snake)

        # Draw a segment of the Hamiltonian cycle for reference
        draw_hamiltonian_cycle(snake)

        # Render Scores
        render_text_with_background(f"Score: {current_score}", WHITE, (GRID_WIDTH * CELL_SIZE + 10, 10))
        render_text_with_background(f"High Score: {high_score}", WHITE, (GRID_WIDTH * CELL_SIZE + 10, 40))

        # Render Debug Information
        if debug_mode:
            render_text_with_background(f"Boost Active: {snake.boost_active}", YELLOW, (GRID_WIDTH * CELL_SIZE + 10, 70))
            render_text_with_background(f"Boost Duration: {snake.boost_duration}", YELLOW, (GRID_WIDTH * CELL_SIZE + 10, 100))
            render_text_with_background(f"Boost Cooldown: {snake.boost_cooldown}", YELLOW, (GRID_WIDTH * CELL_SIZE + 10, 130))
            render_text_with_background(f"AI Epsilon: {agent.epsilon:.2f}", YELLOW, (GRID_WIDTH * CELL_SIZE + 10, 160))
            render_text_with_background(f"Cycle Index: {snake.cycle_index}", YELLOW, (GRID_WIDTH * CELL_SIZE + 10, 190))

        pygame.display.flip()

        if done:
            episode += 1
            agent.update_epsilon()
            if episode % agent.target_update == 0:
                agent.update_target_network()
            # Reset the game
            snake = Snake()
            food = Food(snake.positions)
            state = get_state(snake, food)
            action = agent.get_action(state)
            direction, use_boost = parse_action(action)
            if use_boost:
                snake.activate_boost()
            # Reset previous distance
            previous_distance = manhattan_distance(snake.head(), food.position)

    pygame.quit()
    sys.exit()

if __name__ == "__main__":
    main()