Is this code good, how can you improve it?

import re
import os
import cv2
import logging
import pydirectinput
import torch
import torch.nn as nn
import torch.optim as optim
import random
import pygetwindow as gw
import numpy as np
import subprocess
import pyautogui
import torchvision.models as models
import time
from collections import namedtuple
from PIL import Image

# Disable failsafe feature of PyAutoGUI
pyautogui.FAILSAFE = False

# Configure logging
LOG_FILE = r"C:\Users\Ninja.000\Downloads\SFD_AI\training_log.txt"
logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Define the DQN model
class DQN(nn.Module):
    def __init__(self, input_size, output_size, use_pretrained=False):
        super(DQN, self).__init__()

        # Use a pre-trained model (e.g., ResNet18) as the feature extractor
        if use_pretrained:
            self.feature_extractor = models.resnet18(pretrained=True)
            self.feature_extractor.fc = nn.Identity()  # Remove the fully connected layer
        else:
            # Define your own convolutional layers for feature extraction
            self.conv1 = nn.Conv2d(in_channels=input_size[0], out_channels=32, kernel_size=8, stride=4)
            self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2)
            self.conv3 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1)

            # Common layers for both cases
            self.fc1 = nn.Linear(64 * 7 * 7, 512)  # Set the correct input size
            self.fc2 = nn.Linear(512, output_size)

    def forward(self, x):
        if hasattr(self, 'feature_extractor'):  # Check if using a pre-trained model
            x = self.feature_extractor(x)
            x = x.view(x.size(0), -1)  # Flatten the output
        else:
            x = x / 255.0  # Normalize pixel values
            x = torch.as_tensor(x, dtype=torch.float32)  # Remove clone().detach()
            x = x.permute(0, 3, 1, 2)  # Change the order of dimensions (N, H, W, C) to (N, C, H, W)
            x = self.conv1(x)
            x = torch.relu(x)
            x = self.conv2(x)
            x = torch.relu(x)
            x = self.conv3(x)
            x = torch.relu(x)

            # Print the shape of the tensor at this point
            print("Shape after convolutional layers:", x.shape)

            x = torch.flatten(x, start_dim=1)  # Flatten the output

            # Update the fully connected layers with the correct input size
            self.fc1 = nn.Linear(x.size(1), 512)
            self.fc2 = nn.Linear(512, output_size)

            x = self.fc1(x)
            x = torch.relu(x)
            x = self.fc2(x)
        return x

# Define the process frame
def process_frame(frame_path):
    try:
        # Read the frame using OpenCV
        frame = cv2.imread(frame_path)

        # Check if the frame is not None (i.e., successfully loaded)
        if frame is None:
            raise FileNotFoundError(f"Frame not found at path: {frame_path}")

        # Implement any necessary preprocessing here (e.g., resizing, cropping, converting to grayscale)
        # For now, we'll just resize the frame to match the input_size
        processed_frame = cv2.resize(frame, (650, 650))

        return processed_frame
    except Exception as e:
        logger.error(f"Error processing frame {frame_path}: {str(e)}")
        raise

# Define the replay buffer
class ReplayBuffer:
    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, transition):
        if len(self.memory) < self.capacity:
            self.memory.append(None)
        self.memory[self.position] = transition
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

# Define a named tuple for transitions
Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state', 'done'))

# Define the DQN agent
class DQNAgent:
    def __init__(self, input_size, output_size, gamma=0.99, epsilon=1.0, epsilon_decay=0.995, epsilon_min=0.01):
        self.model = DQN(input_size, output_size)
        self.target_model = DQN(input_size, output_size)
        self.optimizer = optim.Adam(self.model.parameters(), lr=0.001)
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_decay = epsilon_decay
        self.epsilon_min = epsilon_min
        self.replay_buffer = ReplayBuffer(10000)

    def select_action(self, state):
        if np.random.rand() < self.epsilon:
            return np.random.randint(len(self.model.fc2.weight))
        else:
            with torch.no_grad():
                q_values = self.model(torch.tensor(state).float())
                return torch.argmax(q_values).item()

    def train(self, batch_size):
        if len(self.replay_buffer) < batch_size:
            return

        transitions = self.replay_buffer.sample(batch_size)
        batch = Transition(*zip(*transitions))

        state_batch = torch.tensor(np.stack(batch.state)).float()
        action_batch = torch.tensor(batch.action).long()
        reward_batch = torch.tensor(batch.reward).float()
        next_state_batch = torch.tensor(np.stack(batch.next_state)).float()
        done_batch = torch.tensor(batch.done).float()

        q_values = self.model(state_batch).gather(1, action_batch.unsqueeze(1))
        next_q_values = self.target_model(next_state_batch).max(1)[0].detach()
        expected_q_values = reward_batch + (1 - done_batch) * self.gamma * next_q_values

        loss = nn.MSELoss()(q_values, expected_q_values.unsqueeze(1))
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

        self.epsilon = max(self.epsilon * self.epsilon_decay, self.epsilon_min)

    def update_target_model(self):
        self.target_model.load_state_dict(self.model.state_dict())

# Function to capture the game state from the screen
def get_state():
    screenshot = pyautogui.screenshot()
    state = np.array(screenshot)
    return state

# Function to check if the game is open
def is_game_open():
    game_window = gw.getWindowsWithTitle('Superfighters Deluxe')
    return bool(game_window)

# Function to simulate actions based on the agent's output
def simulate_action(action):
    if action == 0:
        # Randomly choose between Jump / ClimbLadder and Move to the Left or Move to the Right
        if np.random.rand() < 0.5:
            pydirectinput.keyDown('up')  
            pydirectinput.keyDown('left')
            logger.info("Simulating action: Jump / ClimbLadder and Move to the Left")
        else:
            pydirectinput.keyDown('up')
            pydirectinput.keyDown('right')
            logger.info("Simulating action: Jump / ClimbLadder and Move to the Right")
    elif action == 1:
        pydirectinput.keyDown('down')  # Crouch / SlideDownLadder
        logger.info("Simulating action: Crouch / SlideDownLadder")
    elif action == 2:
        # Move to the left
        pydirectinput.keyDown('left')
        logger.info("Simulating action: Move to the left")
    elif action == 3:
        # Move to the right
        pydirectinput.keyDown('right')
        logger.info("Simulating action: Move to the right")

    time.sleep(0.1)

    keys_to_release = ['up', 'down', 'left', 'right', 'd', 'a', 's', 'shift']
    keys_to_release.extend(map(str, range(1, 6)))
    keys_to_release.extend(['left', 'right', 'down', 'f'])

    for key in keys_to_release:
        pydirectinput.keyUp(key)

def main():
    global output_size  # Declare output_size as global

    input_size = (3, 650, 650)
    output_size = 4
    use_pretrained = True
    agent = DQNAgent(input_size, output_size, use_pretrained)
    agent.model.to(device)
    agent.target_model.to(device)

    NUM_EPISODES = 100
    MAX_EPISODE_LENGTH = 1000
    BATCH_SIZE = 32
    EPISODE_DELAY = 10
    LOG_FILE = r"C:\Users\Ninja.000\Downloads\SFD_AI\training_log.txt"
    MODEL_SAVE_PATH = r"C:\Users\Ninja.000\Downloads\SFD_AI\dqn_model.pth"
    BEST_MODEL_SAVE_PATH = r"C:\Users\Ninja.000\Downloads\SFD_AI\best_dqn_model.pth"

    # Add a console handler to the logger
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.INFO)
    logger.addHandler(console_handler)

    try:
        # Load the model and its parameters if it exists
        if os.path.exists(MODEL_SAVE_PATH):
            agent.model.load_state_dict(torch.load(MODEL_SAVE_PATH))

            # Extract episode number from the model path
            match = re.search(r'\d+', os.path.basename(MODEL_SAVE_PATH))
            episode_start = int(match.group()) if match else 0
            logger.info(f"Resuming training from Episode {episode_start + 1}")
        else:
            episode_start = 0
            logger.info("Starting training from Episode 1")

        # Load the best model parameters if it exists
        if os.path.exists(BEST_MODEL_SAVE_PATH):
            agent.best_model.load_state_dict(torch.load(BEST_MODEL_SAVE_PATH))
            best_reward = max(np.load(LOG_FILE), default=float('-inf'))
            logger.info(f"Resuming training with the best reward: {best_reward}")
        else:
            best_reward = float('-inf')
            logger.info("Starting training with no best reward information")

        episode_rewards = []

        for episode in range(episode_start, episode_start + NUM_EPISODES):
            logger.info(f"Episode {episode + 1}")

            # Training phase: observe gameplay without taking actions
            total_reward = 0
            experiences = []  # Collect experiences for training
            action = None  # Initialize action variable

            frames = []  # Collect frames for processing
            for t in range(6, MAX_EPISODE_LENGTH, 6):
                frame_path = f"C:\\Users\\Ninja.000\\Downloads\\SFD_AI\\gameplay\\Tutorial\\frame_{t}.jpg"
                logger.info(f"Observing frame: {frame_path}")

                # Process the frame
                state = process_frame(frame_path)
                frames.append(state)

            for t in range(len(frames) - 1):
                # Capture the next frame for the experience
                next_state = frames[t + 1]

                # Simulate random action for exploration
                action = np.random.randint(output_size)
                simulate_action(action)

                # Check if the player picked up a weapon
                weapon_picked_up = did_pick_up_weapon(state, next_state)

                # Check if an enemy was killed
                enemy_killed = is_enemy_killed(state, next_state)

                # Replace the placeholder reward and termination condition with actual logic
                reward, done = calculate_reward_and_check_termination(state, next_state, weapon_picked_up, enemy_killed)
                total_reward += reward
                experiences.append(Transition(state, action, reward, next_state, done))
                state = next_state  # Update the current state

            episode_rewards.append(total_reward)
            logger.info(f"Episode {episode + 1}, Total Reward: {total_reward}")

            # Train the agent using collected experiences for the entire episode
            agent.replay_buffer.extend(experiences)
            agent.train(BATCH_SIZE)
            agent.update_target_model()

            # Save the model parameters at the end of each episode
            torch.save(agent.model.state_dict(), MODEL_SAVE_PATH)

            # Check if this episode achieved the best reward
            if total_reward > best_reward:
                best_reward = total_reward
                torch.save(agent.model.state_dict(), BEST_MODEL_SAVE_PATH)

            # Save the best reward information
            np.save(LOG_FILE, [best_reward])

            # Clear experiences for the next episode
            experiences.clear()

            time.sleep(EPISODE_DELAY)

    except KeyboardInterrupt:
        logger.info("Training interrupted by user. Saving the model before exiting...")
        torch.save(agent.model.state_dict(), MODEL_SAVE_PATH)
        np.save(LOG_FILE, [best_reward])  # Save the best reward information
    except Exception as e:
        logger.error(f"An error occurred: {str(e)}")
    finally:
        exit()