Implementing a paper

arianhf: <p>I am trying to implement <a href="https://bmcbioinformatics.biomedcentral.com/articles/10.1186/s12859-017-1700-2" rel="nofollow noopener">this</a> paper.</p>
<p>I have written the following code but since this is my first try, I am not sure about the code I have written.</p>
<pre><code class="lang-auto">class myDataset(Dataset):
    ...
    def __getitem__(self, idx):
        self.item = self.sequences_1[idx] + self.sequences_2[idx]
        return self.item, self.labels[idx], self.vectorize

def collate_fn(data):
    items, labels, vectorizes = zip(*data)
    labels = torch.tensor(labels)
    items = torch.tensor(items)
    return items.double(), labels.double(), vectorizes

def fetch_dataloader(vectorize, data):
    shuffle_dataset = True
    random_seed= 42
    batch_size = 4

    dataset_size = len(data)
    indices = list(range(len(data)))
    split = 7000

    dataset = myDataset(vectorize, data)
    trainset, valset = random_split(dataset, [dataset_size-split , split])

    train_loader = DataLoader(trainset, batch_size=10, shuffle=True, collate_fn=collate_fn)
    validation_loader = DataLoader(valset, batch_size=10, shuffle=True, collate_fn=collate_fn)

    dataloaders = {"train":train_loader, "test":validation_loader}

    return dataloaders


class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder, self).__init__()
        self.encoder_softmax = nn.Sequential(
            nn.Linear(686, 256),
            nn.ReLU(True),
            nn.Linear(256, 2),
            nn.Softmax()
        )

    def forward(self, x):
        x = self.encoder_softmax(x)
        return x

def train():
    data = InitializeDataLoader(num_test=7000)

    dataloaders = fetch_dataloader('tri', data)
    train_dl = dataloaders['train']
    test_dl = dataloaders['test']
    
    net = autoencoder()

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(device)

    net = net.to(device)
    iterations = 10
    learning_rate = 0.98
    criterion = nn.CrossEntropyLoss()

    optimizer = torch.optim.Adam(
        net.parameters(), lr=learning_rate, weight_decay=1e-5)

    
    for epoch in range(iterations):
        loss = 0.0
        print("train_dl len: ", len(train_dl))
        
        # net.train()
        for i, data in enumerate(train_dl, 0):
            inputs, labels, vectorize = data
            
            labels = labels.long().to(device)
            inputs = inputs.float().to(device)
            optimizer.zero_grad()
            outputs = net(inputs)
            
            train_loss = criterion(outputs, labels)
            
            train_loss.backward()
            optimizer.step()
            
            loss += train_loss.item()

        
        loss = loss / len(train_dl)
        print("epoch : {}/{}, loss = {:.6f}".format(epoch + 1, iterations, loss))

        # create checkpoint variable and add important data
        checkpoint = {
            'epoch': epoch + 1,
            'state_dict': net.state_dict(),
            'optimizer': optimizer.state_dict(),
        }
        
        # save checkpoint
        checkpoint_path = "./checkpoint/current_checkpoint.pt"
        best_model_path =  "./best_model/best_model.pt"
        save_ckp(checkpoint, False, checkpoint_path, best_model_path)

    return net
</code></pre>
<p>I have emitted some parts of code that didn’t seem important. I would be happy to share them too, if it helps.</p>

ptrblck: <p><code>nn.CrossEntropyLoss</code> expects raw logits as the model outputs, so you should remove the last <code>softmax</code> layer in your model.</p>
<p>Do you have any other doubts regarding a specific part of the code?</p>

arianhf: <p>I’m not sure about usage of float, and double casts. anytime I run and it complains, I change them <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=9" title=":slight_smile:" class="emoji" alt=":slight_smile:"> when should I use float? when should I use double? Also, how do I now test this model?</p>

ptrblck: <p>The default data type of <code>float</code> (float32) and <code>double</code> (float64) will trade more precision for performance, which seems to be used in e.g. simulations etc., where the 32bit precision is not enough.</p>
<p>Similar to your training loop, you could load your test data and test the model with it.<br>
Have a look at <a href="https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#test-the-network-on-the-test-data">this tutorial</a> for more information.</p>