Multivariate Input data LSTM training(val_loss is too low)

username: DonghunP
cooked: <p>Hello friend. I’m training multivariate input data using LSTM network.</p>
<p>You can see my data type below.</p>
<ul>
<li>I’m training 64 Input size data.</li>
<li>The ‘Evaluation data’ includes Attack data labeled with ‘T’ containing intermediate random bit values.</li>
<li>I trained only normal data(only labeled as ‘R’) and evaluate data which has random bit and labeled as ‘T’  as shown in the data example below.</li>
</ul>
<pre><code class="lang-auto">'''
                            Evaluation Data                     ↓Label
0111100000000000000000000010000100000000000000000000000000000000R
0111100000000000000000000010000100000000000000000000000000000000R
1000010011111101001011001000100100000000000000000000000000000000T  #&lt;--- Attack
0111100000000000000000001010000100000000000000000000000000000000R
0111100000000000000000000100000100000000000000000000000000000000R
...
0111100000000000000000001000000100000000000000000000000000000000R
0111100000000000000000000011000100000000000000000000000000000000R
0010101100100010100010100011010100000000000000000000000000000000T  #&lt;--- Attack
0111100000000000000000000100000100000000000000000000000000000000R
0111100000000000000000001100000100000000000000000000000000000000R
...
0111100100000000000000000011000000000000000000000000000000000000R
0111100100000000000000000010000000000000000000000000000000000000R
0101101001000010100100000000110100000000000000000000000000000000T #&lt;--- Attack
0111100100000000000000001011000000000000000000000000000000000000R
0111100100000000000000001010000000000000000000000000000000000000R
...
0111011000000000000000000010111100000000000000000000000000000000R
0110110111000111100000010010011100000000000000000000000000000000T  #&lt;--- Attack
0111011000000000000000000001111100000000000000000000000000000000R
'''
</code></pre>
<pre><code class="lang-auto">def sliding_windows(data, seq_length):
    '''
        Separate 64bit by 1bit and make it as list
    '''
    x = []
    y = []
    L = len(data)
    for i in range(L - seq_length-1):
        _x = [list(map(int,e[:-1])) for e in data[i:(i+seq_length)]]
        _y = list(map(int, data[i+seq_length][:-1]))
        x.append(_x)
        y.append(_y)

    return np.array(x),np.array(y)

def sliding_windows_eval(data, seq_length):
    '''
        Separate 64bit by 1bit and make it as list
    '''
    x = []
    y = []
    gt = []
    L = len(data)
    for i in range(L - seq_length):
        _x = [list(map(int,e[:-2])) for e in data[i:(i+seq_length)]]
        _y = list(map(int, data[i+seq_length][:-2]))
        x.append(_x)
        y.append(_y)
        gt.append(data[i + seq_length][-2])

    return np.array(x),np.array(y), np.array(gt)

class LSTM(nn.Module):
    def __init__(self):
        super(LSTM, self).__init__()
        self.num_classes = 64  
        self.num_layers = 1    
        self.input_size = 64   
        self.hidden_size = 512 

        self.lstm = nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size,
                            num_layers=self.num_layers, batch_first=True)

        self.fc = nn.Linear(self.hidden_size, self.num_classes)

    def forward(self, x):
        h_0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size)) #([1, batch, 512])
        c_0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size)) #([1, batch, 512])

        # Propagate input through LSTM
        output, (h_out, _) = self.lstm(x, (h_0, c_0)) # x : [batch, seq, 64], output : [batch, seq, 64]
                                                      # h_out : [1, batch, 512]
        h_out = h_out.view(-1, self.hidden_size)      
        out = self.fc(h_out)
        return out

# Train ------------------------------------------------------------------------------------------ #
with open('./log/%s_normal.log' % ID, 'r') as f:
    training_data = f.readlines()

    x, y = sliding_windows(training_data, seq_length)
    trainX = torch.Tensor(np.array(x))
    trainY = torch.Tensor(np.array(y))
    dataset = TensorDataset(trainX, trainY)  

    dataloader = DataLoader(dataset, batch_size=1000, shuffle=False)

    # optimizer = torch.optim.Adam(lstm.parameters(), lr=learning_rate)
    optimizer = torch.optim.SGD(lstm.parameters(), lr=learning_rate)

    for epoch in range(num_epochs):
        for batch_idx, samples in enumerate(dataloader):
            x_samples, y_samples = samples

            outputs = lstm(x_samples)
            optimizer.zero_grad()

            # obtain the loss function
            loss = criterion(outputs, y_samples)

            loss.backward()

            optimizer.step()
        if epoch % 10 == 0:
            print("Epoch: %d, loss: %1.5f" % (epoch, loss.item()))

    PATH = 'D:/002_5_OTIDS/log/{}_normal.pt'.format(ID)
    torch.save(lstm.state_dict(), PATH)
# ----------------------------------------------------------------------------------------------- #

# Test ------------------------------------------------------------------------------------------ #
PATH = 'D:/002_5_OTIDS/log/{}_normal.pt'.format(ID)

lstm.load_state_dict(torch.load(PATH))

val_losses = []
with open('./log_fuzzy/%s_fuzzy.log' % ID, 'r') as f:
    test_data = f.readlines()
    x, y, gt = sliding_windows_eval(test_data, seq_length)
    testX = torch.Tensor(np.array(x))
    testY = torch.Tensor(np.array(y))
    dataset = TensorDataset(testX, testY)  
    dataloader = DataLoader(dataset, batch_size=1, shuffle=False)

    with torch.no_grad(): # [21.05.11]  check one by one
        for idx, samples in enumerate(dataloader):
            x_samples, y_samples = samples
            lstm.eval()
            outputs = lstm(x_samples)
            val_loss = criterion(outputs, y_samples)
            val_losses.append(val_loss)
            if gt[idx] == 'T':
                print(val_loss)
# ----------------------------------------------------------------------------------------------- #
</code></pre>
<p>My question is</p>
<ol>
<li>
<p>I got val_loss <strong>0.2xxxx</strong> which has label ‘T’ and <strong>0.1xxxx</strong> which has label ‘R’<br>
This doesn’t seem to have big difference.  Is this due to the nature of my data?<br>
or My training is wrong??</p>
</li>
<li>
<p>Can you check my code a little bit? Is there a wrong?</p>
</li>
</ol>
<p>Thanks ahead.<br>
Have a good day.</p>