将以下代码改为TD3算法 
import gym
import torch
import torch.nn as nn
import torch.optim as optim
import random
import numpy as np
from collections import deque
import matplotlib.pyplot as plt

# 定义 DQN 网络结构
class DQN(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(DQN, self).__init__()
        self.fc = nn.Sequential(
            nn.Linear(input_dim, 256),
            nn.ReLU(),
            nn.Linear(256, 256),
            nn.ReLU(),
            nn.Linear(256, output_dim)
        )

    def forward(self, x):
        return self.fc(x)

# 超参数
GAMMA = 0.99                   # 折扣因子
LEARNING_RATE = 1e-3           # 学习率
BATCH_SIZE = 64                # 批大小
BUFFER_SIZE = 50000            # Replay Buffer 大小
EPSILON_START = 1.0            # 初始探索率
EPSILON_END = 0.05             # 最小探索率
EPSILON_DECAY = 0.995          # 探索率衰减
TARGET_UPDATE_FREQ = 10        # 目标网络更新频率（回合数）
SOFT_UPDATE_TAU = 0.005        # Soft Update 权重参数
EPISODES = 22000                 # 训练总回合
TEST_EPISODES = 5              # 测试回合数
TEST_EPSILON = 0.05            # 测试探索率

# 经验回放缓存
class ReplayBuffer:
    def __init__(self, size):
        self.buffer = deque(maxlen=size)
    
    def push(self, state, action, reward, next_state, done):
        self.buffer.append((state, action, reward, next_state, done))
    
    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size)
        states, actions, rewards, next_states, dones = zip(*batch)
        return (
            torch.tensor(states, dtype=torch.float32),
            torch.tensor(actions, dtype=torch.int64),
            torch.tensor(rewards, dtype=torch.float32),
            torch.tensor(next_states, dtype=torch.float32),
            torch.tensor(dones, dtype=torch.float32)
        )
    
    def size(self):
        return len(self.buffer)

# Epsilon-Greedy 策略
def select_action(state, policy_net, epsilon, action_dim, testing=False):
    """Choose action based on epsilon-greedy strategy."""
    if random.random() < (TEST_EPSILON if testing else epsilon):
        return random.randint(0, action_dim - 1)
    with torch.no_grad():
        return policy_net(state).argmax().item()

# Soft Update 目标网络
def soft_update(target_net, policy_net, tau):
    for target_param, param in zip(target_net.parameters(), policy_net.parameters()):
        target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)

# 更新 Q-Network
def optimize_model(policy_net, target_net, replay_buffer, optimizer):
    if replay_buffer.size() < BATCH_SIZE:
        return
    # 从 Replay Buffer 中采样
    states, actions, rewards, next_states, dones = replay_buffer.sample(BATCH_SIZE)
    
    # 当前 Q 值和目标 Q 值
    current_q_values = policy_net(states).gather(1, actions.unsqueeze(1)).squeeze(1)
    next_q_values = target_net(next_states).max(1)[0].detach()
    expected_q_values = rewards + GAMMA * next_q_values * (1 - dones)
    
    # 计算损失
    loss = nn.MSELoss()(current_q_values, expected_q_values)
    
    # 更新策略网络参数
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

# 训练和测试代码
def train_and_test():
    env = gym.make("CartPole-v1")
    state_dim = env.observation_space.shape[0]
    action_dim = env.action_space.n

    # 初始化网络
    policy_net = DQN(state_dim, action_dim)
    target_net = DQN(state_dim, action_dim)
    target_net.load_state_dict(policy_net.state_dict())
    target_net.eval()

    optimizer = optim.Adam(policy_net.parameters(), lr=LEARNING_RATE)
    replay_buffer = ReplayBuffer(BUFFER_SIZE)
    
    # 训练变量
    epsilon = EPSILON_START
    best_score = -float('inf')  # 最高得分
    rewards = []                # 每回合的奖励

    # 训练部分
    for episode in range(EPISODES):
        state = env.reset()
        if isinstance(state, tuple):
            state = state[0]
        state = torch.tensor(state, dtype=torch.float32)
        total_reward = 0

        while True:
            # 选择动作
            action = select_action(state, policy_net, epsilon, action_dim)
            step_result = env.step(action)
            if len(step_result) == 5:
                next_state, reward, done, truncated, _ = step_result
                done = done or truncated
            else:
                next_state, reward, done, _ = step_result

            if isinstance(next_state, tuple):
                next_state = next_state[0]
            next_state = torch.tensor(next_state, dtype=torch.float32)

            total_reward += reward
            replay_buffer.push(state.numpy(), action, reward, next_state.numpy(), done)
            state = next_state

            optimize_model(policy_net, target_net, replay_buffer, optimizer)

            if done:
                break

        # 更新 epsilon
        epsilon = max(EPSILON_END, epsilon * EPSILON_DECAY)

        # Soft Update 目标网络
        soft_update(target_net, policy_net, SOFT_UPDATE_TAU)

        # 保存最高分模型
        if total_reward > best_score:
            best_score = total_reward
            torch.save(policy_net.state_dict(), 'best_dqn_model.pth')
            print(f"New Best Score: {best_score} in Episode {episode}")

        rewards.append(total_reward)

        print(f"Episode {episode}, Total Reward: {total_reward}")

    # 可视化训练曲线
    plt.plot(rewards, label='Episode Rewards')
    plt.axhline(best_score, color='r', linestyle='--', label=f'Best Score: {best_score}')
    plt.xlabel('Episode')
    plt.ylabel('Total Reward')
    plt.legend()
    plt.show()

    # 测试保存的模型
    model = DQN(state_dim, action_dim)
    model.load_state_dict(torch.load('best_dqn_model.pth'))
    model.eval()

    print("=== Testing the Best Model ===")
    for test_episode in range(TEST_EPISODES):
        state = env.reset()
        if isinstance(state, tuple):
            state = state[0]
        state = torch.tensor(state, dtype=torch.float32)
        total_reward = 0

        while True:
            env.render()
            action = select_action(state, model, epsilon=0.0, action_dim=action_dim, testing=True)
            step_result = env.step(action)
            if len(step_result) == 5:
                next_state, reward, done, truncated, _ = step_result
                done = done or truncated
            else:
                next_state, reward, done, _ = step_result
            
            if isinstance(next_state, tuple):
                next_state = next_state[0]
            next_state = torch.tensor(next_state, dtype=torch.float32)

            total_reward += reward
            state = next_state

            if done:
                print(f"Test Episode {test_episode}, Total Reward: {total_reward}")
                break

    env.close()

if __name__ == "__main__":
    train_and_test()