Actor-critic / inplace operation error

username: sho
cooked: <p>I’m making a code(actor-critic frame / Exactly counterfactual multi-agent(COMA).<br>
and anyway</p>
<p>I can’t fix following error</p>
<pre><code class="lang-auto">Traceback (most recent call last):
  File "C:/Users/sueng/OneDrive - SNU/FJSP_MARL/COMA.py", line 330, in &lt;module&gt;
    agents.train()
  File "C:\Users\sueng\OneDrive - SNU\FJSP_MARL\COMA.py", line 216, in train
    actor_loss.backward()
  File "C:\Users\sueng\anaconda3\envs\FJSP\lib\site-packages\torch\tensor.py", line 166, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "C:\Users\sueng\anaconda3\envs\FJSP\lib\site-packages\torch\autograd\__init__.py", line 99, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [1, 36]], which is output 0 of SoftmaxBackward, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).
</code></pre>
<pre><code class="lang-auto">import torch
import torch.nn as nn
import torch.nn.functional as F
from FJSP import RL_ENV

from torch.distributions import Categorical
from copy import deepcopy


class Memory:
    def __init__(self, agent_num, action_dim):
        self.agent_num = agent_num
        self.action_dim = action_dim
        self.states = []
        self.actions = []
        self.actions_last = []
        self.observations = []
        self.pi = [[] for _ in range(agent_num)]
        self.reward = []
        self.done = [[] for _ in range(agent_num)]
    def get(self):
        actions = torch.tensor(self.actions)
        observations = self.observations
        states = self.states
        actions_last = self.actions_last
        pi = []
        for i in range(self.agent_num):
            pi.append(torch.cat(self.pi[i]).view(len(self.pi[i]), self.action_dim))
        reward = torch.tensor(self.reward)
        done = self.done
        return states, actions, observations, pi, reward, actions_last, done
    def clear(self):
        self.actions = []
        self.observations = []
        self.states = []
        self.actions_last = []
        self.pi = [[] for _ in range(self.agent_num)]
        self.reward = []
        self.done = [[] for _ in range(self.agent_num)]


class Actor(nn.Module):
    def __init__(self, obs_dim, action_dim, hidden_dim):
        super(Actor, self).__init__()
        self.fc1 = nn.Linear(obs_dim+action_dim, 64)
        self.GRU_layer = nn.GRUCell(64, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, action_dim)
        #self.fc3 = nn.Linear(45, action_dim)
        self.ReLU = nn.ReLU()
        self.Softmax = nn.Softmax(dim = 1)
    def forward(self, obs_a_cat, hidden_last):
        x = self.ReLU(self.fc1(obs_a_cat))
        gru_out = self.GRU_layer(x, hidden_last)
        utility = self.fc2(gru_out)
        action_prob = self.Softmax(utility)
        #utility = x.detach()
        return action_prob, gru_out, utility

class Critic(nn.Module):
    def __init__(self, agent_num, state_dim, obs_dim, action_dim):
        super(Critic, self).__init__()

        input_dim = (agent_num-1) *action_dim + state_dim + obs_dim +agent_num + action_dim
        #     action_dim + state_dim + obs_dim +   + action_dim
        # other_agent_actions, states, obs, agent_id, agent_last_action

        self.fc1 = nn.Linear(input_dim, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, action_dim)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        output = self.fc3(x)
        return output


class COMA:
    def __init__(self, agent_num, state_dim, obs_dim, action_dim, hidden_dim, lr_c, lr_a, gamma, target_update_steps):
        self.agent_num = agent_num
        self.state_dim = state_dim
        self.obs_dim = obs_dim
        self.action_dim = action_dim
        self.hidden_dim = hidden_dim

        self.gamma = gamma

        self.target_update_steps = target_update_steps

        self.memory = Memory(agent_num, action_dim)

        self.actors = [Actor(obs_dim, action_dim, hidden_dim) for _ in range(agent_num)]
        self.critic = Critic(agent_num, state_dim, obs_dim, action_dim)

        self.critic_target = Critic(agent_num, state_dim, obs_dim, action_dim)
        self.critic_target.load_state_dict(self.critic.state_dict())

        self.actors_optimizer = [torch.optim.Adam(self.actors[i].parameters(), lr=lr_a) for i in range(agent_num)]
        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=lr_c)

        self.encoded_action = np.eye(action_dim)
        self.encoded_id = np.eye(agent_num)
        self.count = 0

    def get_actions(self, observations, avail_actions, action_last, hidden_last):
        #print(observations.shape)
        observations = np.array(observations).reshape(-1, self.obs_dim)
        action_last = np.array(action_last).reshape(-1, self.action_dim)
        obs_u_cat = np.concatenate([observations, action_last], axis = 1)
        #print(obs_u_cat.shape)


        actions = []
        hiddens = []
        utilities = []


        for i in range(self.agent_num):

            obs_u_i = torch.Tensor(obs_u_cat[i]).reshape(1, -1)
            hidden_i = torch.Tensor(hidden_last[i]).reshape(1, -1)

            #print(obs_u_i.shape, hidden_i.shape)

            #print(obs_u_i.shape, hidden_i.shape, self.obs_dim+self.action_dim)
            #hidden_i = hidden_i.view(1, -1)
            #
            #with torch.autograd.set_detect_anomaly(True):
            dist, hidden, utility = self.actors[i](obs_u_i, hidden_i)
            self.memory.pi[i].append(dist)
            avail_action = avail_actions[i]
            mask = torch.from_numpy(np.array(avail_action)).reshape(1,-1)
            dist[mask==0] = 0

            action = Categorical(dist).sample()
            actions.append(action.item())
            hiddens.append(hidden)

            utilities.append(utility.detach().numpy().tolist())

        return actions, hiddens, utilities

    def train(self):
        actor_optimizer = self.actors_optimizer
        critic_optimizer = self.critic_optimizer
        states, actions, observations, pi, reward, actions_last, done = self.memory.get()
        states = torch.Tensor(states).view(-1, self.state_dim)

        encoded_actions = [self.encoded_action[act].tolist() for act in actions]


        actions_last = np.reshape(actions_last, (self.agent_num, -1, self.action_dim))
        observations = np.reshape(observations, (self.agent_num, -1, self.obs_dim))



        print("????????????????")

        for i in range(self.agent_num):
                # train actor
            other_agent_actions = deepcopy(encoded_actions)
            other_agent_actions = [delete_index(actions, i) for actions in other_agent_actions]
            agent_last_action = actions_last[i]
            obs = observations[i]
            agent_id = [self.encoded_id[i] for _ in range(len(reward))]
            input_critic = self.build_input_critic(other_agent_actions, states, obs, agent_id, agent_last_action)

            Q_target = self.critic_target(input_critic).detach() # target network는 (직접적인) 학습을 수행하지 않는 파라미터이므로 detach()를 써야함

            action_taken = actions.type(torch.long)[:, i].reshape(-1, 1)

            """ pi((u_a_1,..... u_a-) * q(s, (u_a_1,..... u_a-)) + 
                pi((u_a_2,..... u_a-) * q(s, (u_a_2,..... u_a-)) +...
                pi((u_a_|U|,.....u_a_) * q(s, (u_a_|U|,....u_a_))"""

            Q_taken_target = torch.gather(Q_target, dim=1, index=action_taken).squeeze()

            # train critic

            Q = self.critic(input_critic)
            action_taken = actions.type(torch.long)[:, i].reshape(-1, 1)
            Q_taken = torch.gather(Q, dim=1, index=action_taken).squeeze()



            y_i = []
            for t in range(len(reward)):
                if done[t]:
                    y_i.append(reward[t] + self.gamma * Q_taken_target[t + 1].item())
                else:
                    y_i.append(reward[t])

            #y_i = torch.Tensor([ if done[t] else reward[t] for t in range(len(reward))])
            y_i = torch.Tensor(y_i)
            #print(y_i.shape, Q_taken.shape)
            critic_loss = torch.mean((y_i.detach() - Q_taken) ** 2)
            critic_optimizer.zero_grad()
            critic_loss.backward()
            torch.nn.utils.clip_grad_norm_(self.critic.parameters(), 5)
            critic_optimizer.step()



            Q_taken_target_clone = Q_taken_target.clone()
            Q_target_clone = Q_target.clone()
            baseline = torch.sum(pi[i] * Q_target_clone, dim=1)

            advantage = Q_taken_target_clone.detach() - baseline # advantage 계산
            log_pi = torch.log(torch.gather(pi[i], dim=1, index=action_taken).squeeze())
            actor_loss = - torch.mean(advantage.detach() * log_pi)
            print(actor_loss)

            actor_optimizer[i].zero_grad()
            #with torch.autograd.set_detect_anomaly(True):

            actor_loss.backward()
            torch.nn.utils.clip_grad_norm_(self.actors[i].parameters(), 5)


            actor_optimizer[i].step()


        if self.count == self.target_update_steps:
            self.critic_target.load_state_dict(agents.critic.state_dict())
            self.count = 0
        else:
            self.count += 1

        self.memory.clear()


    def build_input_critic(self, other_agent_actions, state, obs, agent_id, agent_last_action): #agent_id, observations, actions):
        other_agent_actions = torch.Tensor(other_agent_actions).view(-1, (self.agent_num-1)*self.action_dim)
        state = torch.Tensor(state).view(-1, self.state_dim)
        obs = torch.Tensor(obs).view(-1, self.obs_dim)
        agent_id = torch.Tensor(agent_id).view(-1, self.agent_num)
        agent_last_action = torch.Tensor(agent_last_action).view(-1, self.action_dim)
        input_critic = torch.cat([other_agent_actions, state, obs, agent_id, agent_last_action], axis = 1)
        return input_critic

def delete_index(input, idx):
    del input[idx]
    return input

import matplotlib.pyplot as plt
import numpy as np

from COMA import COMA


def moving_average(x, N):
    return np.convolve(x, np.ones((N,)) / N, mode='valid')


if __name__ == "__main__":
    # Hyperparameters
    #with torch.autograd.set_detect_anomaly(True):
    env = RL_ENV()
    env_info = env.get_env_info()

    agent_num = env_info["n_agents"]
    obs_dim = env_info['obs_shape'] #+ agent_num
    state_dim = env_info['state_shape']
    action_dim = env_info["n_actions"]
    hidden_dim = 32

    gamma = 0.99
    lr_a = 0.0001
    lr_c = 0.005

    target_update_steps = 10

    # agent initialisation

    agents = COMA(agent_num, state_dim, obs_dim, action_dim, hidden_dim, lr_c, lr_a, gamma, target_update_steps)


    env.render()

    episode_reward = 0
    episodes_reward = []

    # training loop

    n_episodes = 10000
    episode = 0

    for episode in range(n_episodes):
        env = RL_ENV()
        env.render()
        episode_reward = 0
        makespan = 0
        actions_last = env.last_action
        hidden_last = torch.zeros((agent_num, hidden_dim))
        done = False
        step = 0
        while done==False:

            obs = env.get_obs()
            state = env.get_state()


            agents.memory.actions_last.append(actions_last)
            avail_actions = np.array(env.get_avail_actions())
            actions, hidden_last, utility = agents.get_actions(obs, avail_actions, actions_last, hidden_last)

            utility = utility

            indices = torch.LongTensor(actions).view(-1, 1)

            utility = torch.Tensor(utility).view(agent_num, -1)

            utility = torch.gather(utility, 1, indices)

            reward, done, actions = env.step(actions, utility)
            reward = reward / 100
            agents.memory.states.append(state)
            agents.memory.observations.append(obs)
            agents.memory.actions.append(actions)
            agents.memory.done.append(done)
            agents.memory.reward.append(reward)
            step = step +1
            print(step)


            if done:
                episodes_reward.append(episode_reward)
                episode_reward = 0
                episode = episode +1
                agents.train()

                score = env.env.now
                print(f"episode: {episode}, average reward: {score}")
</code></pre>