How to compute loss with HuggingFace transformers?
Author: h56cho
Pull request: no
Labels: name: wontfix

Hello,

Is it possible to train HuggingFace TransfoXLLMHeadModel on a dataset different than WikiText103, say, on the combined WikiText2 and WikiText103 dataset?

Below are my code:
```js

# Import packages 
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import TransfoXLConfig, TransfoXLTokenizer, TransfoXLLMHeadModel
from transformers import AdamW, WarmupLinearSchedule
import spacy
import torchtext
from torchtext.data.utils import get_tokenizer
from torchtext.data import Field, BPTTIterator, TabularDataset 
import tensorflow as tf
import math
import random
import numpy as np
import pandas as pd 
import time

# set hyperparameters for this experiment
bptt = 30
batch_size = 64
lr = 0.01 # learning rate
criterion = nn.CrossEntropyLoss() # loss criterion

# define tokenizer
en = spacy.load('en')

def Sp_Tokenizer(text): 
    return [tok.text for tok in en.tokenizer(text)]

# define the English text field
TEXT = Field(tokenize = Sp_Tokenizer,
             init_token='< sos >',
             eos_token='< eos >',
             unk_token='< unk >',
             tokenizer_language='en',
             lower=True)

# load the datasets
train_Wiki2, val_Wiki2, test_Wiki2 = torchtext.datasets.WikiText2.splits(TEXT)
train_Wiki103, val_Wiki103, test_Wiki103 = torchtext.datasets.WikiText103.splits(TEXT)

# Define device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# build vocabulary based on the defined field and
# the combined WikiText2 and WikiText103 datasets.
TEXT.build_vocab(train_Wiki2, val_Wiki2, test_Wiki2, 
                 train_Wiki103, val_Wiki103, test_Wiki103)

# set hyperparameter ntokens
ntokens = len(TEXT.vocab.stoi)

## specify the transformer-XL model that we are going to use.
#
# define transformer-XL configuration.
transfoXLconfig = TransfoXLConfig(vocab_size_or_config_json_file = ntokens,
                                  cutoffs = [20000, 40000, 200000], 
                                  d_model = 64, 
                                  d_embed = 64, 
                                  n_head = 16, 
                                  d_head = 64,
                                  n_layer = 5,
                                  attn_type = 0,
                                  dropout = 0.1, 
                                  output_hidden_states = True,
                                  output_attentions = True)

# define the transformer-XL model based on the specified configuration.
model = TransfoXLLMHeadModel(transfoXLconfig)

# add new tokens to the embeddings of our model
model.resize_token_embeddings(ntokens)

# define BPTTiterators
#
train_iter, val_iter, test_iter = BPTTIterator.splits(
            (train_Wiki2, val_Wiki2, test_Wiki2),
            batch_size = batch_size,
            bptt_len= bptt,
            sort_key=lambda x: len(x.text),
            sort_within_batch = True,
            shuffle = False,
            device= device,
            repeat=False)

train = next(iter(train_iter))
val = next(iter(train_iter))
test = next(iter(test_iter))

```

and now I am trying to write the train function but I am not sure how exactly I should proceed.
Below is what I tried:

```js

# define the hyperparameters for running the train function.
train = train 
optimizer = AdamW(model.parameters())
scheduler = WarmupLinearSchedule(optimizer = optimizer, 
                                 warmup_steps = 200, 
                                 t_total = 1000,
                                 last_epoch = -1)
model.train()

# define the train function
def train(model, train, bptt, criterion, optimizer, scheduler, ntokens, log_interval):

    # initialize total_loss to 0
    total_loss = 0
    # measure the computation time
    start_time = time.time()
    # number of tokens in the vocabulary
    ntokens = ntokens 

    for i in range(train.text.size()[1]):
        
        batch = i
        input_ids, targets = train.text[:,i], train.target[:,i]
        input_ids = torch.tensor(input_ids.tolist()).unsqueeze(0) 
        targets = torch.tensor(targets.tolist()).unsqueeze(0)
        optimizer.zero_grad()

        # I intend this 'output' to be the final output of the Transformer-XL....
        output = model(input_ids) 
       #... to execute this line
        loss = criterion(output.view(-1, ntokens), targets)

        loss.backward()

        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
        optimizer.step()
```

But I don't think the line `loss = criterion(output.view(-1, ntokens), targets)` shouldn't work since the line `output = model(input_ids)` does not actually give out the final output from the model, but it rather gives out (according to the HuggingFace documentation) prediction_scores, mems, attention, etc. How can I train TransfoXLLMHeadModel on a dataset different than just WikiText103?

Thank you,

Comments:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.