Implement Keras Stateful-LSTM model to Pytorch

username: 20am847
cooked: <p>Hi, I am a kind of Newb in pytorch <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=9" title=":slight_smile:" class="emoji" alt=":slight_smile:"></p>
<p>What I’m trying to do is a <strong>time series prediction model.</strong><br>
After many trials and errors, I found the Keras code I wanted and tried to apply it to the pytorch.</p>
<p>The main point of the Keras model is set to <strong>stateful = True</strong>, so I also used the hidden state and cell state values ​​of the previous mini-batch <strong>without initializing the values</strong> ​​of the hidden state and cell state.</p>
<p>But my code does not drop the loss value at all, and the prediction value seems very strange. I have no idea what’s wrong. What I want to do is to start with the most basic model, so I set the value of look-ahead day and look-back day to 1.(It is the same for the Keras code.)  Feature dimension is also 1.</p>
<p>So a shape of input_X should be (1, batch_size, 1)</p>
<p>Anyone, please take a look my code and point out why loss value does not reduce.  The summary of keras model and pytorch code is below.</p>
<pre><code class="lang-auto"># Keras Model

def build_model(self):
    # first add input to hidden1
    self.model.add(LSTM(
        units=120,
        batch_input_shape=(672,1,1),
        stateful=True)
    self.model.add(Dropout(0.1))
    
    # add dense layer with output dimension to get output for one time_step
    self.model.add(Dense(units=1))

    # Repeat for look_ahead steps to get outputs for look_ahead timesteps.
    self.model.add(RepeatVector(1)

    # add activation
    self.model.add(Activation("linear"))

    # compile model and print summary
    self.model.compile(loss="MSE", optimizer=Adam(lr=0.02,decay= .99))

    return self.model
</code></pre>
<p><em>And my pytorch code</em></p>
<pre><code class="lang-auto">Xtrain = torch.load('X_train.pt')
ytrain = torch.load('y_train.pt')

batch_size = 672

remained = len(Xtrain) % 672  # len(Xtrain) = 22846
new_len = len(Xtrain) - remained # new_len = 22176

# In a stateful network, you should only pass inputs with a number of samples that can be divided by the batch size
X_train = Xtrain[:new_len]
y_train = ytrain[:new_len]

# transpose for input shape
X_train = np.transpose(X_train, (1, 0, 2))
batch_X_train = X_train.reshape(-1,672,1)

y_train = np.transpose(y_train, (1, 0, 2))
batch_y_train = y_train.reshape(-1,672,1)

print(batch_X_train.shape) # (33, 672, 1)
print(batch_y_train.shape) # (33, 672, 1)

trainX_tensor = torch.FloatTensor(batch_X_train)
trainY_tensor = torch.FloatTensor(batch_y_train)

trainData = TensorDataset(trainX_tensor, trainY_tensor)
</code></pre>
<pre><code class="lang-auto">class LSTM(nn.Module):

    def __init__(self, input_dim, hidden_size, out_dim, num_layers, batch_size, dropout, device):
        super(LSTM, self).__init__()

        self.batch_size = batch_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.dropout_p = dropout
        self.device = device

        self.lstm = nn.LSTM(input_dim, hidden_size, num_layers=num_layers)
        self.fc1 = nn.Linear(hidden_size, hidden_size//2)
        self.dropout = nn.Dropout(p = self.dropout_p)
        self.fc2 = nn.Linear(hidden_size//2,out_dim)

    def forward(self, x, hidden, cell):
        lstm_out, (hidden, cell) = self.lstm(x, (hidden, cell))

        out = self.fc1(lstm_out)
        out = F.relu(out)
        out = self.dropout(out)
        out = self.fc2(out)

        return out, hidden, cell
</code></pre>
<pre><code class="lang-auto">epoch = 5
input_dim = 1
out_dim = 1
hidden_size = 120
num_layers = 1
batch_size = 672
dropout = 0.1
learning_rate = .02

loss_function = nn.MSELoss()

model = LSTM(input_dim, hidden_size, out_dim, num_layers, batch_size, dropout, device).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=.99)

# initialize hidden and cell state
def hidden_state_init(stack_num, bsz, hidden):
    hw = torch.empty(stack_num, bsz, hidden)
    cw = torch.empty(stack_num, bsz, hidden)
    h_state = torch.nn.init.xavier_uniform_(hw)
    c_state = torch.nn.init.xavier_uniform_(cw)
    return h_state,c_state
</code></pre>
<pre><code class="lang-auto">for epoch_num in range(epoch):
    model.train()
    y_prediction = []
    loss_list = []
    average_loss = 0
    h_state, c_state=hidden_state_init(num_layers, batch_size,hidden_size)
    
    for idx, data in enumerate(trainData):

        X, y = data
        batch_size = X.shape[0]
        X = X.reshape(-1, batch_size, 1)

        # For stateful
        h_state = h_state.detach()
        c_state = c_state.detach()

        optimizer.zero_grad()

        y_pred, h_state, c_state  = model(X, h_state, c_state)
        y_prediction.append(y_pred)

        loss = loss_function(y_pred, y)

        loss.backward()
        optimizer.step()

        average_loss += (loss.item() / 33)
        loss_list.append(loss)

    print(average_loss)
   1.0038617777101921
   1.0039293639587632
   1.003980219364166
   1.0042506131258877
   1.0043293851794615


</code></pre>
<p>I’m not sure what I have to do now. Any suggestion would be really appreciated.</p>