# Buggy code
def learn(self, batch: Batch, **kwargs: Any) -> Dict[str, float]:
    weight = batch.pop("weight", 1.0)
    target_q = batch.returns.flatten()
    act = to_torch(batch.act[:, np.newaxis], device=target_q.device, dtype=torch.long)

    # critic 1
    current_q1 = self.critic1(batch.obs).gather(1, act).flatten()
    td1 = current_q1 - target_q
    critic1_loss = (td1.pow(2) * weight).mean()

    self.critic1_optim.zero_grad()
    critic1_loss.backward()
    self.critic1_optim.step()

    # critic 2
    current_q2 = self.critic2(batch.obs).gather(1, act).flatten()
    td2 = current_q2 - target_q
    critic2_loss = (td2.pow(2) * weight).mean()

    self.critic2_optim.zero_grad()
    critic2_loss.backward()
    self.critic2_optim.step()
    batch.weight = (td1 + td2) / 2.0  # prio-buffer

    # actor
    dist = self(batch).dist
    entropy = dist.entropy()
    with torch.no_grad():
        current_q1a = self.critic1(batch.obs)
        current_q2a = self.critic2(batch.obs)
        q = torch.min(current_q1a, current_q2a)
    actor_loss = -(self._alpha * entropy + (dist.probs * q).sum(dim=-1)).mean()
    self.actor_optim.zero_grad()
    actor_loss.backward()
    self.actor_optim.step()

    if self._is_auto_alpha:
        log_prob = entropy.detach() - self._target_entropy
        alpha_loss = (self._log_alpha * log_prob).mean()
        self._alpha_optim.zero_grad()
        alpha_loss.backward()
        self._alpha_optim.step()
        self._alpha = self._log_alpha.detach().exp()

    self.sync_weight()

    result = {
        "loss/actor": actor_loss.item(),
        "loss/critic1": critic1_loss.item(),
        "loss/critic2": critic2_loss.item(),
    }
    if self._is_auto_alpha:
        result["loss/alpha"] = alpha_loss.item()
        result["alpha"] = self._alpha.item()  # type: ignore

    return result


# Traceback (TypeError)
Epoch #1:   0%|          | 0/1 [00:21<?, ?it/s]
Traceback (most recent call last):
File "D:/PycharmProjects/Stable-BaselineTrading/Tianshou/TD3.py", line 85, in <module>
save_fn=lambda p: torch.save(p.state_dict(), save_dir))
File "D:\PycharmProjects\Stable-BaselineTrading\Tianshou\Trainer\offpolicy.py", line 110, in offpolicy_trainer
losses = policy.update(batch_size, train_collector.buffer)
File "C:\Users\zkx74\Anaconda3\envs\RL\lib\site-packages\tianshou\policy\base.py", line 147, in update
result = self.learn(batch, *args, **kwargs)
File "C:\Users\zkx74\Anaconda3\envs\RL\lib\site-packages\tianshou\policy\modelfree\td3.py", line 123, in learn
critic1_loss = (td1.pow(2) * weight).mean()
File "C:\Users\zkx74\Anaconda3\envs\RL\lib\site-packages\torch\tensor.py", line 480, in __array__
return self.numpy()
TypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

# Fixed code
def learn(self, batch: Batch, **kwargs: Any) -> Dict[str, float]:
    weight = batch.pop("weight", 1.0)
    target_q = batch.returns.flatten()
    act = to_torch(batch.act[:, np.newaxis], device=target_q.device, dtype=torch.long)

    # critic 1
    current_q1 = self.critic1(batch.obs).gather(1, act).flatten()
    td1 = current_q1 - target_q
    critic1_loss = (td1.pow(2) * weight).mean()

    self.critic1_optim.zero_grad()
    critic1_loss.backward()
    self.critic1_optim.step()

    # critic 2
    current_q2 = self.critic2(batch.obs).gather(1, act).flatten()
    td2 = current_q2 - target_q
    critic2_loss = (td2.pow(2) * weight).mean()

    self.critic2_optim.zero_grad()
    critic2_loss.backward()
    self.critic2_optim.step()
    batch.weight = (td1 + td2) / 2.0  # prio-buffer

    # actor
    dist = self(batch).dist
    entropy = dist.entropy()
    with torch.no_grad():
        current_q1a = self.critic1(batch.obs)
        current_q2a = self.critic2(batch.obs)
        q = torch.min(current_q1a, current_q2a)
    actor_loss = -(self._alpha * entropy + (dist.probs * q).sum(dim=-1)).mean()
    self.actor_optim.zero_grad()
    actor_loss.backward()
    self.actor_optim.step()

    if self._is_auto_alpha:
        log_prob = -entropy.detach() + self._target_entropy
        alpha_loss = -(self._log_alpha * log_prob).mean()
        self._alpha_optim.zero_grad()
        alpha_loss.backward()
        self._alpha_optim.step()
        self._alpha = self._log_alpha.detach().exp()

    self.sync_weight()

    result = {
        "loss/actor": actor_loss.item(),
        "loss/critic1": critic1_loss.item(),
        "loss/critic2": critic2_loss.item(),
    }
    if self._is_auto_alpha:
        result["loss/alpha"] = alpha_loss.item()
        result["alpha"] = self._alpha.item()  # type: ignore

    return result