Loss not decreasing in LSTM implemented by myself

username: xiaomaguohe123
cooked: <p>I have implemented the lstm model based on the pytorch nn.module，the Forward propagation  result of the model are the same as  the official results.However, when i use my own LSTM model to complete the  binary classification mission, the training loss is around 0.69. my code is as follows,<br>
<span class="hashtag">#自定义LSTM_Cell内部的结构:即这个结构自定义一个Cell结构的层</span>，四个门，两个输出,<br>
<span class="hashtag">#学习参数为W和b</span>,其余参数均可以通过计算得到</p>
<h1>LSTM_cell自定义</h1>
<h1>LSTM_cell自定义</h1>
<p>class LSTM_Cell(nn.Module):<br>
def <strong>init</strong>(self, input_size, hidden_size):<br>
super(LSTM_Cell, self).<strong>init</strong>()<br>
self.input_size = input_size<br>
self.hidden_size = hidden_size</p>
<pre><code>	# parameter of input gate
	self.w_ii = Parameter(Tensor(hidden_size, input_size))
	self.b_ii = Parameter(Tensor(hidden_size, 1))
	self.w_hi = Parameter(Tensor(hidden_size, hidden_size))
	self.b_hi = Parameter(Tensor(hidden_size, 1))

	# parameter of forget gate
	self.w_if = Parameter(Tensor(hidden_size, input_size))
	self.b_if = Parameter(Tensor(hidden_size, 1))
	self.w_hf = Parameter(Tensor(hidden_size, hidden_size))
	self.b_hf = Parameter(Tensor(hidden_size, 1))

	# parameter of output gate
	self.w_io = Parameter(Tensor(hidden_size, input_size))
	self.b_io = Parameter(Tensor(hidden_size, 1))
	self.w_ho = Parameter(Tensor(hidden_size, hidden_size))
	self.b_ho = Parameter(Tensor(hidden_size, 1))

	# parameter of cell memory
	self.w_ic = Parameter(Tensor(hidden_size, input_size))
	self.b_ic = Parameter(Tensor(hidden_size, 1))
	self.w_hc = Parameter(Tensor(hidden_size, hidden_size))
	self.b_hc = Parameter(Tensor(hidden_size, 1))
	self.reset_weights()
    
def reset_weights(self):
	stdv = 1.0 / math.sqrt(self.hidden_size)
	for weight in self.parameters():
		init.uniform_(weight, -stdv, stdv)

# forward propagation of LSTM cell
def forward(self,inputs, h, c):
	# input gate output
	i = torch.sigmoid(self.w_ii @ inputs + self.b_ii + self.w_hi @ h + self.b_hi)
	# forget gate output
	f = torch.sigmoid(self.w_if @ inputs + self.b_if + self.w_hf @ h + self.b_hf)
	# output gate output
	o = torch.sigmoid(self.w_io @ inputs + self.b_io + self.w_ho @ h + self.b_ho)
	# candidate state
	g = torch.tanh(self.w_ic @ inputs + self.b_ic + self.w_hc @ h + self.b_hc)
	# cell memory of current time step
	c_current = f * c + i * g
	# hidden state of current time step
	h_current = o * torch.tanh(c_current)
	return h_current, c_current

# initialization of hidden state and cell memory
def init_hidden(self, batch_size, hidden_size):
	h_init = Variable(torch.randn(batch_size,hidden_size).t())
	c_init = Variable(torch.randn(batch_size,hidden_size).t())
	return h_init, c_init
</code></pre>
<h1>LSTM model 多层自定义层（n&gt;=1）</h1>
<p>class LSTM_layer(nn.Module):<br>
def <strong>init</strong>(self,input_size,hidden_size,layer_num):<br>
super(LSTM_layer,self).<strong>init</strong>()<br>
self.input_size = input_size<br>
self.hidden_size = hidden_size<br>
self.num_layers = layer_num<br>
# LSTM Cell list<br>
self._all_layers = []<br>
for i in range(self.num_layers):<br>
layer_name = ‘cell{}’.format(i)<br>
if i==0:<br>
cell = LSTM_Cell(self.input_size, self.hidden_size)<span class="hashtag">#定义多层cell</span><br>
cell.reset_weights()<br>
setattr(self, layer_name, cell)<span class="hashtag">#setattr</span> 用于给类设置属性<br>
self._all_layers.append(cell)<br>
else:<br>
cell = LSTM_Cell(self.hidden_size, self.hidden_size)<span class="hashtag">#定义多层cell</span> 第二层的输入为第一层的h输出<br>
cell.reset_weights()<br>
setattr(self, layer_name, cell)<br>
self._all_layers.append(cell) <span class="hashtag">#定义一个层结构</span>，将定义的层都加入到层结构中<br>
def forward(self, inputs):<br>
# store internal state (hidden state, cell memory)<br>
internal_state=[]<br>
outputs=[] #<br>
for t in range(inputs.size(0)):<br>
x_step=inputs[t].t() <span class="hashtag">#当前输入</span><br>
for layer in range(self.num_layers):<br>
layer_name = ‘cell{}’.format(layer)<br>
if t==0:<br>
batch_size = inputs[t].size()[0]<br>
h, c= getattr(self, layer_name).init_hidden(batch_size=batch_size,hidden_size=self.hidden_size)<br>
internal_state.append((h, c))<br>
(h, c)= internal_state[layer]<br>
x_step, c_new= getattr(self,layer_name)(x_step,h,c)<br>
internal_state[layer] = (x_step, c_new)<br>
outputs.append(x_step.t().unsqueeze(0))<span class="hashtag">#在第一维增加维度</span>，采用函数unsqueeze(0)<br>
outputs = torch.cat(outputs, dim=0)<span class="hashtag">#将三个张量合并为一个张量</span><br>
return  outputs,(x_step, c_new)</p>
<p><span class="hashtag">#定义通用模型网络结构</span><br>
class my_model(nn.Module):<br>
def <strong>init</strong>(self,input_size,hidden_size,layer_num,output_size):<br>
super(my_model, self).<strong>init</strong>()<br>
self.input_size=input_size<br>
self.hidden_size=hidden_size<br>
self.layer_num=layer_num<br>
<span class="hashtag">#self</span>.bidirection=bidirection<br>
self.output_size=output_size<br>
self.lstm =LSTM_layer(self.input_size, self.hidden_size,self.layer_num)<br>
self.linear=nn.Linear(self.hidden_size,self.output_size)<br>
def forward(self,inputs):<br>
r_out,(h_final, c_final)= self.lstm(inputs)<br>
out =self.linear(r_out[-1, :, :])  <span class="hashtag">#self</span>.linear(h_final.t()) torch.sigmoid(self.linear(h_final.t()))<br>
return   out</p>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
model=my_model(14,50,1,2)<span class="hashtag">#input_size</span>,hidden_size,layer_num,output_size<br>
<span class="hashtag">#for</span> name, parameter in model.named_parameters():<br>
#	print(name, parameter)<br>
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)   # optimize all parameters<br>
<span class="hashtag">#optimizer</span> =torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)<span class="hashtag">#leanring_rate</span><br>
criterion =nn.CrossEntropyLoss()      <span class="hashtag">#定义损失函数</span> nn.MSELoss()<br>
epoch = 0<br>
for epoch in range(num_epochs):<br>
for data in train_loader:<br>
img, target = data<br>
img = img.view(M,-1,14) <span class="hashtag">#lenth_size</span>,batch,input_size<br>
target=target.squeeze(1)<br>
img = img.squeeze(0)<br>
img = Variable(img)<br>
target = Variable(target)<br>
out = model(img)<br>
loss = criterion(out, target)<br>
print_loss = loss.data.item()<br>
optimizer.zero_grad()<br>
loss.backward()<br>
optimizer.step()<br>
epoch+=1<br>
if epoch%10==0:<br>
print(‘epoch: {}, loss: {:.4}’.format(epoch, print_loss))</p>