Hi, previously you did a wonderful job of implementing a q actor critic algorithm for me. Here is the implementation : class QActorCritic(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(QActorCritic, self).__init__()
        self.policy_net = nn.Sequential(
            nn.Linear(num_inputs, 64),
            nn.ReLU(),
            nn.Linear(64, 64),
            nn.ReLU(),
            nn.Linear(64, num_outputs),
            nn.Softmax(dim=1)
        )
        
        self.q_net = nn.Sequential(
            nn.Linear(num_inputs, 64),
            nn.ReLU(),
            nn.Linear(64, 64),
            nn.ReLU(),
            nn.Linear(64, num_outputs)
        )        
        
    def forward(self, x):
        raise NotImplementedError

    def get_action_probs(self, x):
        x = torch.from_numpy(x).float().unsqueeze(0).to(device)
        action_probs = self.policy_net(x)
        return action_probs

    def get_q_values(self, x):
        x = torch.from_numpy(x).float().unsqueeze(0).to(device)
        q_values = self.q_net(x)
        return q_values and def train(env, device, model, lr=1e-3, num_episodes=500, gamma=0.99):
    policy_optimizer = optim.Adam(model.policy_net.parameters(), lr=lr)
    q_optimizer = optim.Adam(model.q_net.parameters(), lr=lr)
    total_rewards = []

    for episode in range(num_episodes):
        state = env.reset()
        episode_reward = 0
        
        while True:
            action_probs = model.get_action_probs(state)
            action_distr = Categorical(action_probs)
            action = action_distr.sample().item()

            next_state, reward, done, _ = env.step(action)

            next_action_probs = model.get_action_probs(next_state)
            next_action_distr = Categorical(next_action_probs)
            next_action = next_action_distr.sample().item()

            with torch.no_grad():
                target = reward + gamma * model.get_q_values(next_state)[0, next_action]

            policy_loss = -action_distr.log_prob(torch.tensor([action]).to(device)) * (target - model.get_q_values(state)[0, action])
            q_loss = (target - model.get_q_values(state)[0, action]).pow(2).to(device)

            policy_optimizer.zero_grad()
            policy_loss.backward()
            policy_optimizer.step()

            q_optimizer.zero_grad()
            q_loss.backward()
            q_optimizer.step()

            state = next_state
            episode_reward += reward
        
            if done:
                break
        total_rewards.append(episode_reward)
        
        if episode % 100 == 0:
          print(f"Episode: {episode}, reward: {episode_reward}")
    
    return total_rewards. This here is the calling function : # Bipedal Walker environment
env = gym.make('BipedalWalker-v3')
num_inputs = env.observation_space.shape[0]
num_outputs = env.action_space.shape[0]
q_actor_critic = QActorCritic(num_inputs, num_outputs).to(device)
train_rewards = train(env,device, q_actor_critic, num_episodes=1000, lr=1e-4)

print('Bipedal Walker Training results:')
plot_rewards('BipedalWalker-v3', 'Training', train_rewards) .                                                                                                                                                                                                               
But I'm getting the following error. ---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-32-1bda7f217336> in <cell line: 6>()
      4 num_outputs = env.action_space.shape[0]
      5 q_actor_critic = QActorCritic(num_inputs, num_outputs).to(device)
----> 6 train_rewards = train(env,device, q_actor_critic, num_episodes=1000, lr=1e-4)
      7 
      8 print('Bipedal Walker Training results:', plot_rewards('BipedalWalker-v3', 'Training', train_rewards))

6 frames
/usr/local/lib/python3.10/dist-packages/gym/envs/box2d/bipedal_walker.py in step(self, action)
    531             self.joints[3].motorSpeed = float(SPEED_KNEE * np.clip(action[3], -1, 1))
    532         else:
--> 533             self.joints[0].motorSpeed = float(SPEED_HIP * np.sign(action[0]))
    534             self.joints[0].maxMotorTorque = float(
    535                 MOTORS_TORQUE * np.clip(np.abs(action[0]), 0, 1)

TypeError: 'int' object is not subscriptable