Time series Prediction Transformer

tjrqls1017: <p>import torch<br>
import torch.nn as nn<br>
import math</p>
<p>class PositionalEncoding(nn.Module):<br>
def <strong>init</strong>(self, d_model, max_len=15):<br>
super(PositionalEncoding, self).<strong>init</strong>()<br>
pe = torch.zeros(max_len, d_model)<br>
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)<br>
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))<br>
pe[:, 0::2] = torch.sin(position * div_term)<br>
pe[:, 1::2] = torch.cos(position * div_term)<br>
pe = pe.unsqueeze(0).transpose(0, 1)<br>
self.register_buffer(‘pe’, pe)</p>
<pre><code>def forward(self, x):
    return x + self.pe[:x.size(0), :]
</code></pre>
<p>class TransformerPredictor(nn.Module):<br>
def <strong>init</strong>(self, input_dim, output_dim, d_model, nhead, num_layers, dim_feedforward, dropout=0.1, max_seq_length=15, device=‘cuda’):<br>
super(TransformerPredictor, self).<strong>init</strong>()<br>
self.device = device<br>
self.d_model = d_model<br>
self.input_embedding = nn.Linear(input_dim, d_model).to(device)<br>
self.pos_encoder = PositionalEncoding(d_model, max_seq_length).to(device)<br>
encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, dropout=dropout, activation=‘relu’).to(device)<br>
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers, enable_nested_tensor=False).to(device)<br>
decoder_layer = nn.TransformerDecoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, dropout=dropout, activation=‘relu’).to(device)<br>
self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=num_layers).to(device)<br>
self.fc_layers = nn.Sequential(<br>
nn.Linear(d_model, d_model // 2),<br>
nn.ReLU(),<br>
nn.Linear(d_model // 2, d_model // 4),<br>
nn.ReLU(),<br>
nn.Linear(d_model // 4, d_model // 8),<br>
nn.ReLU(),<br>
nn.Linear(d_model // 8, d_model // 16),<br>
nn.ReLU(),<br>
nn.Linear(d_model // 16, output_dim)<br>
).to(device)<br>
self._reset_parameters()</p>
<pre><code>def forward(self, src, tgt=None, max_len=15):
    src = src.permute(1, 0, 2)
    src = self.input_embedding(src)
    src = self.pos_encoder(src)
    memory = self.transformer_encoder(src)

    if tgt is None:
        tgt = torch.zeros((max_len, src.size(1), self.d_model), device=self.device)
        tgt[0, :, :] = memory[-1, :, :].unsqueeze(0)

        outputs = []
        for i in range(max_len):
            tgt_mask = self.generate_square_subsequent_mask(i+1).to(self.device)
            tgt_emb = self.pos_encoder(tgt[:i+1, :, :])
            output = self.transformer_decoder(tgt_emb, memory, tgt_mask=tgt_mask)
            output_for_next_step = output[-1, :, :]
            if i &lt; max_len - 1:
                tgt[i+1, :, :] = output_for_next_step
            output_transformed = self.fc_layers(output_for_next_step.unsqueeze(0))
            outputs.append(output_transformed)

        outputs = torch.stack(outputs, dim=1).squeeze(2)
        return outputs
    else:
        sos_token = memory[-1, :, :].unsqueeze(0)
        tgt = tgt.permute(1, 0, 2)
        tgt = self.input_embedding(tgt)
        tgt = self.pos_encoder(tgt)
        tgt = torch.cat([sos_token, tgt[:-1,:]], dim=0)
        tgt_mask = self.generate_square_subsequent_mask(tgt.size(0)).to(self.device)
        output = self.transformer_decoder(tgt, memory, tgt_mask=tgt_mask)
        final_output = self.fc_layers(output)
        return final_output.permute(1, 0, 2)

def _reset_parameters(self):
    for p in self.parameters():
        if p.dim() &gt; 1:
            nn.init.xavier_uniform_(p)

def generate_square_subsequent_mask(self, sz):
    mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
    return mask
</code></pre>
<p>I wrote the code for Transformer time-seris prediction for multiple step prediction.<br>
But the model not performs well.</p>
<p>Traning loss is very low when target is given, but when target is not given, it predicts almost same value or very wrong value</p>