C:\Users\mryt\work\Flappy-bird-rl\agent_cuda.py:49: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single n
umpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at C:\cb\pytorch_1000000000000\work\torch\csrc\utils\tensor_new.cpp:264.)
  batch_state = torch.FloatTensor(batch_state).to(self.model)
Traceback (most recent call last):
  File "C:\Users\mryt\work\Flappy-bird-rl\game2.py", line 283, in <module>
    main()
  File "C:\Users\mryt\work\Flappy-bird-rl\game2.py", line 225, in main
    loss = agent.replay(batch_size=64)  # Adjust batch size as needed
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\mryt\work\Flappy-bird-rl\agent_cuda.py", line 49, in replay
    batch_state = torch.FloatTensor(batch_state).to(self.model)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: to() received an invalid combination of arguments - got (DQN), but expected one of:
 * (torch.device device, torch.dtype dtype, bool non_blocking, bool copy, *, torch.memory_format memory_format)
 * (torch.dtype dtype, bool non_blocking, bool copy, *, torch.memory_format memory_format)
 * (Tensor tensor, bool non_blocking, bool copy, *, torch.memory_format memory_format)

import os
import sys
import pygame
import random
import torch
import numpy as np
import matplotlib.pyplot as plt
from collections import deque
from agent_cuda import DQNAgent  # Updated agent with Double DQN
from parameters import lr, gamma, epsilon, epsilon_decay, buffer_size, penalty, epsilon_min

render = False

if not render:
    # Use the 'dummy' video driver for headless mode
    os.environ["SDL_VIDEODRIVER"] = "dummy"

pygame.init()

# Screen dimensions
SCREEN_WIDTH = 400
SCREEN_HEIGHT = 600

# Colors
RED = (255, 0, 0)
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
BLUE = (0, 0, 255)

# Game constants
BIRD_SIZE = 35
FLAP_STRENGTH = -13
GRAVITY = 1
PIPE_SPEED = 5
PIPE_WIDTH = 70
PIPE_GAP = 220
PIPE_FREQUENCY = 90  # Frames between pipes

class Bird:
    def __init__(self):
        self.x = 50
        self.y = SCREEN_HEIGHT // 2
        self.velocity = 0
        self.size = BIRD_SIZE

    def flap(self):
        self.velocity = FLAP_STRENGTH

    def update(self):
        self.velocity += GRAVITY
        self.y += self.velocity

    def get_rect(self):
        return pygame.Rect(self.x, self.y, self.size, self.size)

    def draw(self, screen):
        if render:
            pygame.draw.rect(screen, BLUE, self.get_rect())

class Pipe:
    def __init__(self):
        self.gap = PIPE_GAP
        self.top_height = random.randint(50, SCREEN_HEIGHT - self.gap - 50)
        self.top = pygame.Rect(SCREEN_WIDTH, 0, PIPE_WIDTH, self.top_height)
        self.bottom = pygame.Rect(SCREEN_WIDTH, self.top_height + self.gap, PIPE_WIDTH, SCREEN_HEIGHT - self.top_height - self.gap)

    def draw(self, screen):
        if render:
            pygame.draw.rect(screen, WHITE, self.top)
            pygame.draw.rect(screen, WHITE, self.bottom)

    def move(self):
        self.top.x -= PIPE_SPEED
        self.bottom.x -= PIPE_SPEED

    def off_screen(self):
        return self.top.right < 0

def get_state(bird, pipes):
    """
    State includes bird y position, velocity, distance to next pipe,
    top and bottom of next pipe.
    """
    # Find the next pipe
    next_pipe = None
    for pipe in pipes:
        if pipe.top.right > bird.x:
            next_pipe = pipe
            break
    if next_pipe is None:
        # If no pipes are present, provide default values
        distance = SCREEN_WIDTH
        top = SCREEN_HEIGHT // 2 - PIPE_GAP // 2
        bottom = SCREEN_HEIGHT // 2 + PIPE_GAP // 2
    else:
        distance = (next_pipe.top.x + PIPE_WIDTH) - bird.x
        top = next_pipe.top.height
        bottom = next_pipe.bottom.y

    velocity = bird.velocity

    # Normalize state
    distance_norm = distance / SCREEN_WIDTH
    top_norm = top / SCREEN_HEIGHT
    bottom_norm = bottom / SCREEN_HEIGHT
    y_norm = bird.y / SCREEN_HEIGHT
    velocity_norm = velocity / 10  # Assuming max velocity won't exceed 10 in magnitude

    state = np.array([y_norm, velocity_norm, distance_norm, top_norm, bottom_norm], dtype=np.float32)
    return state

def take_action(action, bird):
    if action == 1:  # Flap
        bird.flap()

def main():
    # Initialize the game components
    if render:
        screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
        pygame.display.set_caption("Flappy Bird with DQN")
        font = pygame.font.SysFont(None, 36)
        clock = pygame.time.Clock()
    else:
        screen = None
        font = None
        clock = None

    # Initialize Bird and other game variables
    bird = Bird()
    pipes = []
    pipe_timer = PIPE_FREQUENCY
    score = 0
    game_active = True  # Start the game immediately

    # Initialize DQN Agent
    state_dim = 5
    action_dim = 2  # Flap or do nothing
    agent = DQNAgent(
        state_dim=state_dim,
        action_dim=action_dim,
        lr=lr,
        gamma=gamma,
        epsilon=epsilon,
        epsilon_decay=epsilon_decay,
        buffer_size=buffer_size,
        epsilon_min=epsilon_min,
        target_update_freq=1000  # Adjust as needed
    )

    episode_rewards = []
    episode_scores = []
    moving_avg_scores = []
    num_episodes = int(sys.argv[1]) if len(sys.argv) > 1 else 1000

    for episode in range(1, num_episodes + 1):
        # Reset the game state
        bird = Bird()
        pipes = []
        pipe_timer = PIPE_FREQUENCY
        score = 0
        game_active = True

        episode_reward = 0

        while game_active:
            if render:
                for event in pygame.event.get():
                    if event.type == pygame.QUIT:
                        pygame.quit()
                        sys.exit()

                screen.fill(BLACK)

            # Get current state
            state = get_state(bird, pipes)

            # Select action
            action = agent.act(state)
            take_action(action, bird)

            # Update bird
            bird.update()

            # Manage pipes
            pipe_timer += 1
            if pipe_timer > PIPE_FREQUENCY:
                pipes.append(Pipe())
                pipe_timer = 0
            for pipe in pipes[:]:
                pipe.move()
                if pipe.off_screen():
                    pipes.remove(pipe)
                    score += 1
                    episode_reward += 1  # Reward for passing a pipe

            # Draw pipes and bird
            if render:
                for pipe in pipes:
                    pipe.draw(screen)
                bird.draw(screen)

            # Check for collisions
            done = False
            reward = 0.1  # Small reward for being alive
            bird_rect = bird.get_rect()
            for pipe in pipes:
                if bird_rect.colliderect(pipe.top) or bird_rect.colliderect(pipe.bottom):
                    game_active = False
                    done = True
                    reward = penalty
                    break
            if bird.y <= 0 or bird.y + bird.size >= SCREEN_HEIGHT:
                game_active = False
                done = True
                reward = penalty

            # Get next state
            next_state = get_state(bird, pipes)

            # Remember experience
            agent.remember(state, action, reward, next_state, done)
            episode_reward += reward

            # Update the agent
            loss = agent.replay(batch_size=64)  # Adjust batch size as needed

            if render:
                # Display the score
                score_text = font.render(f"Score: {score}", True, WHITE)
                screen.blit(score_text, (10, 10))

                # Update the display
                pygame.display.flip()
                # Control the frame rate
                clock.tick(60)  # Increased FPS for faster training if rendered

        # Episode finished
        episode_rewards.append(episode_reward)
        episode_scores.append(score)
        moving_avg = np.mean(episode_scores[-100:]) if len(episode_scores) >= 100 else np.mean(episode_scores)
        moving_avg_scores.append(moving_avg)

        # Decay epsilon
        agent.decay_epsilon()

        # Print valuable data
        print(f"Episode {episode}/{num_episodes} | Score: {score} | "
              f"Reward: {episode_reward:.2f} | Avg Score: {moving_avg:.2f} | "
              f"Epsilon: {agent.epsilon:.4f}")

    # Save the trained model
    os.makedirs('models', exist_ok=True)
    existing_models = len(os.listdir('models/'))
    model_path = f"models/flappy{existing_models + 1}.pth"
    agent.save_model(model_path)

    # Plot the rewards and scores
    plt.figure(figsize=(12, 6))

    plt.subplot(1, 2, 1)
    plt.plot(range(len(episode_rewards)), episode_rewards, label='Episode Reward')
    plt.plot(range(len(moving_avg_scores)), moving_avg_scores, label='100-Episode Moving Average')
    plt.title("Episode Rewards")
    plt.xlabel("Episode")
    plt.ylabel("Reward")
    plt.legend()

    plt.subplot(1, 2, 2)
    plt.plot(range(len(episode_scores)), episode_scores, label='Score per Episode')
    plt.plot(range(len(moving_avg_scores)), moving_avg_scores, label='100-Episode Moving Average')
    plt.title("Scores")
    plt.xlabel("Episode")
    plt.ylabel("Score")
    plt.legend()

    plt.tight_layout()
    plt.show()

    # Quit Pygame
    pygame.quit()

if __name__ == "__main__":
    main()


#model.py
import torch
import torch.nn as nn
import numpy as np


class DQN(nn.Module):
    def __init__(self, input_dim, actions):
        super(DQN, self).__init__()
        self.layers = nn.Sequential(
            nn.Linear(input_dim, 128),
            nn.BatchNorm1d(128),
            nn.ReLU(),
            nn.Linear(128, 128),
            nn.BatchNorm1d(128),
            nn.ReLU(),
            nn.Linear(128, actions)
        )

    def forward(self, x):
        return self.layers(x)


# obs = env.observation_space.shape[0]
# actions = env.action_space.n
# model = build_model(obs, actions)


#agent_cuda.py
import torch
import torch.nn as nn
import numpy as np
from collections import deque
import random
from model import DQN  # Ensure DQN model is appropriately defined

class DQNAgent:
    def __init__(self, state_dim, action_dim, lr=0.001, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.05, buffer_size=100000, target_update_freq=1000):
        self.state_dim = state_dim
        self.action_dim = action_dim
        self.lr = lr
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_decay = epsilon_decay
        self.epsilon_min = epsilon_min
        self.memory = deque(maxlen=buffer_size)
        self.target_update_freq = target_update_freq
        self.step_count = 0  # To track when to update target network

        self.model = DQN(state_dim, action_dim)
        self.target_model = DQN(state_dim, action_dim)
        self.target_model.load_state_dict(self.model.state_dict())
        self.target_model.eval()  # Set target network to evaluation mode
        
        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)
        self.loss_fn = torch.nn.MSELoss()

    def act(self, state):
        if np.random.rand() <= self.epsilon:
            return random.choice(range(self.action_dim))  # Supports more than 2 actions
        
        state_tensor = torch.tensor(state, dtype=torch.float32).unsqueeze(0)  # Add batch dimension
        
        with torch.no_grad():
            q_values = self.model(state_tensor)
        return torch.argmax(q_values, dim=1).item()
    
    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    def replay(self, batch_size=128):
        if len(self.memory) < batch_size:
            return None  # Not enough data to update
    
        transitions = random.sample(self.memory, batch_size)
        batch_state, batch_action, batch_reward, batch_next_state, batch_done = zip(*transitions)
    
        batch_state = torch.FloatTensor(batch_state).to(self.model)
        batch_action = torch.LongTensor(batch_action).unsqueeze(1).to(self.model)
        batch_reward = torch.FloatTensor(batch_reward).unsqueeze(1).to(self.model)
        batch_next_state = torch.FloatTensor(batch_next_state).to(self.model)
        batch_done = torch.FloatTensor(batch_done).unsqueeze(1).to(self.model)
    
        # Current Q values from policy network
        current_q_values = self.model(batch_state).gather(1, batch_action)
    
        # Action selection using policy network
        next_actions = self.model(batch_next_state).max(1)[1].unsqueeze(1)
    
        # Action evaluation using target network
        next_q_values = self.target_model(batch_next_state).gather(1, next_actions).detach()
    
        # Compute target Q values
        expected_q_values = batch_reward + (self.gamma * next_q_values * (1 - batch_done))
    
        # Compute loss
        loss = self.loss_fn(current_q_values, expected_q_values)
    
        # Optimize the model
        self.optimizer.zero_grad()
        loss.backward()
        torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0)  # Gradient clipping
        self.optimizer.step()
    
        self.step_count += 1
    
        # Update target network periodically
        if self.step_count % self.target_update_freq == 0:
            self.update_target_network()
    
        return loss.item()
    
    def decay_epsilon(self):
        if self.epsilon > self.epsilon_min:
            self.epsilon = max(self.epsilon_min, self.epsilon * self.epsilon_decay)
                
    def update_target_network(self):
        """ Copies the weights from the policy model to the target model. """
        self.target_model.load_state_dict(self.model.state_dict())
        print("Target network updated.")
                
    def save_model(self, filename):
        torch.save(self.model.state_dict(), filename)
        print(f"Model saved to {filename}")
    
    def load_model(self, filename):
        self.model.load_state_dict(torch.load(filename, map_location=torch.device('cpu')))
        self.target_model.load_state_dict(self.model.state_dict())
        print(f"Model loaded from {filename}")

#model.py
import torch
import torch.nn as nn
import numpy as np
from collections import deque
import random
from model import DQN  # Ensure DQN model is appropriately defined

class DQNAgent:
    def __init__(self, state_dim, action_dim, lr=0.001, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.05, buffer_size=100000, target_update_freq=1000):
        self.state_dim = state_dim
        self.action_dim = action_dim
        self.lr = lr
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_decay = epsilon_decay
        self.epsilon_min = epsilon_min
        self.memory = deque(maxlen=buffer_size)
        self.target_update_freq = target_update_freq
        self.step_count = 0  # To track when to update target network

        self.model = DQN(state_dim, action_dim)
        self.target_model = DQN(state_dim, action_dim)
        self.target_model.load_state_dict(self.model.state_dict())
        self.target_model.eval()  # Set target network to evaluation mode
        
        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)
        self.loss_fn = torch.nn.MSELoss()

    def act(self, state):
        if np.random.rand() <= self.epsilon:
            return random.choice(range(self.action_dim))  # Supports more than 2 actions
        
        state_tensor = torch.tensor(state, dtype=torch.float32).unsqueeze(0)  # Add batch dimension
        
        with torch.no_grad():
            q_values = self.model(state_tensor)
        return torch.argmax(q_values, dim=1).item()
    
    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    def replay(self, batch_size=128):
        if len(self.memory) < batch_size:
            return None  # Not enough data to update
    
        transitions = random.sample(self.memory, batch_size)
        batch_state, batch_action, batch_reward, batch_next_state, batch_done = zip(*transitions)
    
        batch_state = torch.FloatTensor(batch_state).to(self.model)
        batch_action = torch.LongTensor(batch_action).unsqueeze(1).to(self.model)
        batch_reward = torch.FloatTensor(batch_reward).unsqueeze(1).to(self.model)
        batch_next_state = torch.FloatTensor(batch_next_state).to(self.model)
        batch_done = torch.FloatTensor(batch_done).unsqueeze(1).to(self.model)
    
        # Current Q values from policy network
        current_q_values = self.model(batch_state).gather(1, batch_action)
    
        # Action selection using policy network
        next_actions = self.model(batch_next_state).max(1)[1].unsqueeze(1)
    
        # Action evaluation using target network
        next_q_values = self.target_model(batch_next_state).gather(1, next_actions).detach()
    
        # Compute target Q values
        expected_q_values = batch_reward + (self.gamma * next_q_values * (1 - batch_done))
    
        # Compute loss
        loss = self.loss_fn(current_q_values, expected_q_values)
    
        # Optimize the model
        self.optimizer.zero_grad()
        loss.backward()
        torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=1.0)  # Gradient clipping
        self.optimizer.step()
    
        self.step_count += 1
    
        # Update target network periodically
        if self.step_count % self.target_update_freq == 0:
            self.update_target_network()
    
        return loss.item()
    
    def decay_epsilon(self):
        if self.epsilon > self.epsilon_min:
            self.epsilon = max(self.epsilon_min, self.epsilon * self.epsilon_decay)
                
    def update_target_network(self):
        """ Copies the weights from the policy model to the target model. """
        self.target_model.load_state_dict(self.model.state_dict())
        print("Target network updated.")
                
    def save_model(self, filename):
        torch.save(self.model.state_dict(), filename)
        print(f"Model saved to {filename}")
    
    def load_model(self, filename):
        self.model.load_state_dict(torch.load(filename, map_location=torch.device('cpu')))
        self.target_model.load_state_dict(self.model.state_dict())
        print(f"Model loaded from {filename}")