the value of grad is None when I load aggregated parameter

Epiphany: <p>mport torch<br>
import torch.nn as nn<br>
import numpy as np<br>
from Model import rl_net<br>
from torchviz import make_dot</p>
<p>import matplotlib.pyplot as plt</p>
<h1>
<a name="hyper-parameters-1" class="anchor" href="#hyper-parameters-1"></a>Hyper Parameters</h1>
<p>TARGET_REPLACE_ITER = 100<br>
num_action = 3<br>
Memory_capacity = 500<br>
LR = 0.001<br>
EPSILON = 0.9<br>
BATCH_SIZE = 32<br>
GAMMA = 0.95<br>
num_alpha_state = 12<br>
device = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)</p>
<p>class Agent1(object):<br>
def <strong>init</strong>(self):<br>
self.Eval_Net = rl_net().to(device)<br>
self.Target_Net = rl_net().to(device)<br>
self.memory_counter = 0<br>
self.alpha_memory = torch.from_numpy(np.zeros((Memory_capacity, 26))).to(device=device)<br>
self.learn_step_counter_alpha = 0<br>
self.learn_alpha_step_counter = 0<br>
self.rl_net = None<br>
def local_learn(self,localEpoch,localBatchSize,global_parameters):<br>
self.Eval_Net.load_state_dict(global_parameters,strict=True)<br>
loss_func = nn.MSELoss()<br>
opti = torch.optim.Adam(self.Eval_Net.parameters(),lr=LR)<br>
for epoch in range(localEpoch):<br>
self.learn_step_counter_alpha += 1<br>
if self.learn_step_counter_alpha % TARGET_REPLACE_ITER ==0:<br>
self.Target_Net.load_state_dict(self.Eval_Net.state_dict())<br>
sample_index = torch.tensor(np.random.choice(Memory_capacity, localBatchSize),device=device).long()<br>
b_alpha_memory = torch.tensor(self.alpha_memory[sample_index, :],device=device)<br>
b_alpha_s = torch.tensor(b_alpha_memory[:, :num_alpha_state]).float()<br>
b_alpha_a = torch.tensor(b_alpha_memory[:, num_alpha_state:num_alpha_state + 1]).long()<br>
b_alpha_r = torch.tensor(b_alpha_memory[:, num_alpha_state + 1:num_alpha_state + 2])<br>
b_alpha_s_ = torch.tensor(b_alpha_memory[:, -num_alpha_state:]).float()<br>
q_alpha_eval = self.Eval_Net.forward(b_alpha_s).gather(1, b_alpha_a)  # shape (batch, 1)获取动作的reward<br>
q_alpha_next = self.Target_Net.forward(b_alpha_s_).detach() # detach from graph, don’t backpropagate<br>
q_alpha_target = b_alpha_r + GAMMA * q_alpha_next.max(1)[0].view(BATCH_SIZE, 1) # shape (batch, 1)<br>
loss_alpha = torch.tensor(loss_func(q_alpha_eval,q_alpha_target),dtype=float,device=device,requires_grad=True)<br>
opti.zero_grad()<br>
loss_alpha.backward(retain_graph=True)<br>
make_dot(self.Eval_Net(torch.rand(12)))<br>
self.rl_net = self.Target_Net<br>
def store_alpha_transition(self, s, a, r, s_):  # 可复用<br>
s = torch.tensor([s]).to(device=device)<br>
a = torch.tensor([a]).to(device=device)<br>
r = torch.tensor([r]).to(device=device)<br>
s_ = torch.tensor([s_]).to(device=device)<br>
s = torch.squeeze(s,dim=0)<br>
s_ = torch.squeeze(s_, dim=0)<br>
transition = torch.cat((s,a,r,s_))<br>
index = self.memory_counter % Memory_capacity<br>
self.alpha_memory[index, :] = transition<br>
self.memory_counter += 1</p>
<pre><code>def choose_action(self, state):  # 可复用
    if np.random.uniform() &lt; EPSILON:  # greedy

        state = torch.Tensor(state)
        actions_value = self.Target_Net.forward(state)
        action = torch.argmax(actions_value)
            else:  # random
        action = torch.tensor([np.random.randint(0, 3)],device=device)
                return action
def extract_para(self):
    return self.Target_Net.state_dict()

def local_train(self,env,obs,episodes,global_parameters):
    for i in range(episodes):
        action = self.choose_action(obs)
        next_state, reward, done, collision = env.alpha_step(action)
        self.store_alpha_transition(obs,action,reward,next_state)
        obs = next_state
        if self.memory_counter &gt;= Memory_capacity:
            self.local_learn(localEpoch=200,localBatchSize=32,global_parameters=global_parameters)</code></pre>