Hi, You helped me implement a Q Actor Critic algorithm which worked well for Cartpole-v1 , Lunar Lander-v2 but not for Bipedal Walker. So, I'll provide you the entire implementation and I want you to make the necessary modifications to our code. Keep in mind that the code should work for all the three mentioned env's. In my opinion, our current implementation only works for discrete space whereas Bipedal Walker is a continuous space which causing the error. I'm impressed by your work yesterday. You did a great job. Keep up the good work as always. Here is the implementation:                                                                                                                  
class QActorCritic(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(QActorCritic, self).__init__()
        self.policy_net = nn.Sequential(
            nn.Linear(num_inputs, 64),
            nn.ReLU(),
            nn.Linear(64, 64),
            nn.ReLU(),
            nn.Linear(64, num_outputs),
            nn.Softmax(dim=1)
        )
        
        self.q_net = nn.Sequential(
            nn.Linear(num_inputs, 64),
            nn.ReLU(),
            nn.Linear(64, 64),
            nn.ReLU(),
            nn.Linear(64, num_outputs)
        )        
        
    def forward(self, x):
        raise NotImplementedError

    def get_action_probs(self, x):
        x = torch.from_numpy(x).float().unsqueeze(0).to(device)
        action_probs = self.policy_net(x)
        return action_probs

    def get_q_values(self, x):
        x = torch.from_numpy(x).float().unsqueeze(0).to(device)
        q_values = self.q_net(x)
        return q_values                                                                                                                                                                                                                                                                                                   def train(env, device, model, lr=1e-3, num_episodes=500, gamma=0.99):
    policy_optimizer = optim.Adam(model.policy_net.parameters(), lr=lr)
    q_optimizer = optim.Adam(model.q_net.parameters(), lr=lr)
    total_rewards = []

    for episode in range(num_episodes):
        state = env.reset()
        episode_reward = 0
        
        while True:
            action_probs = model.get_action_probs(state)
            action_distr = Categorical(action_probs)
            action = action_distr.sample().item()

            next_state, reward, done, _ = env.step(action)

            next_action_probs = model.get_action_probs(next_state)
            next_action_distr = Categorical(next_action_probs)
            next_action = next_action_distr.sample().item()

            with torch.no_grad():
                target = reward + gamma * model.get_q_values(next_state)[0, next_action]

            policy_loss = -action_distr.log_prob(torch.tensor([action]).to(device)) * (target - model.get_q_values(state)[0, action])
            q_loss = (target - model.get_q_values(state)[0, action]).pow(2).to(device)

            policy_optimizer.zero_grad()
            policy_loss.backward()
            policy_optimizer.step()

            q_optimizer.zero_grad()
            q_loss.backward()
            q_optimizer.step()

            state = next_state
            episode_reward += reward
        
            if done:
                break
        total_rewards.append(episode_reward)
        
        if episode % 100 == 0:
          print(f"Episode: {episode}, reward: {episode_reward}")                                                                                                                                                                                                          
    return total_rewards                                                                                                                                                                                                                                                                                                 # env = gym.make('BipedalWalker-v3')
num_inputs = env.observation_space.shape[0]
num_outputs = env.action_space.shape[0]
q_actor_critic = QActorCritic(num_inputs, num_outputs).to(device)
train_rewards = train(env,device, q_actor_critic, num_episodes=1000, lr=1e-4)

print('Bipedal Walker Training results:')
plot_rewards('BipedalWalker-v3', 'Training', train_rewards)
    
    return total_rewards