Further train bert with next sentence prediction head using tensorflow

Question:
I’m trying to train TFBertForNextSentencePrediction on my own corpus, not from scratch, but rather taking the existing bert model with only a next sentence prediction head and further train it on a specific cuprous of text (pairs of sentences). Then I want to use the model I trained to be able to extract sentence embeddings from the last hidden state for other texts.
Currently the problem I encounter is that after I train the keras model I am not able to extract the hidden states of the last layer before the next sentence prediction head.
Below is the code. Here I only train it on a few sentences just to make sure the code works. Any help will be greatly appreciated.
Thanks, Ayala
import numpy as np
import pandas as pd
import tensorflow as tf
from datetime import datetime
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.callbacks import ModelCheckpoint
from transformers import BertTokenizer, PreTrainedTokenizer, BertConfig, TFBertForNextSentencePrediction
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score

PRETRAINED_MODEL = 'bert-base-uncased'

# set paths and file names
time_stamp = str(datetime.now().year) + "_" + str(datetime.now().month) + "_" + str(datetime.now().day) + "_" + \
                     str(datetime.now().hour) + "_" + str(datetime.now().minute)
model_name = "pretrained_nsp_model"
model_dir_data = model_name + "_" + time_stamp
model_fn = model_dir_data + ".h5"
base_path = os.path.dirname(__file__)
input_path = os.path.join(base_path, "input_data")
output_path = os.path.join(base_path, "output_models")
model_path = os.path.join(output_path, model_dir_data)
if not os.path.exists(model_path):
    os.makedirs(model_path)

# set model checkpoint
checkpoint = ModelCheckpoint(os.path.join(model_path, model_fn), monitor="val_loss", verbose=1, save_best_only=True,
                             save_weights_only=True, mode="min")

# read data
max_length = 512

def get_tokenizer(pretrained_model_name):
    tokenizer = BertTokenizer.from_pretrained(pretrained_model_name)
    return tokenizer

def tokenize_nsp_data(A, B, max_length):
    data_inputs = tokenizer(A, B, add_special_tokens=True, max_length=max_length, truncation=True,
                             pad_to_max_length=True, return_attention_mask=True,
                             return_tensors="tf")
    return data_inputs

def get_data_features(data_inputs, max_length):
    data_features = {}
    for key in data_inputs:
        data_features[key] = sequence.pad_sequences(data_inputs[key], maxlen=max_length, truncating="post",
                                                          padding="post", value=0)
    return data_features

def get_transformer_model(transformer_model_name):
    # get transformer model
    config = BertConfig(output_attentions=True)
    config.output_hidden_states = True
    config.return_dict = True
    transformer_model = TFBertForNextSentencePrediction.from_pretrained(transformer_model_name, config=config)
    return transformer_model

def get_keras_model(transformer_model):
    # get keras model
    input_ids = tf.keras.layers.Input(shape=(max_length,), name='input_ids', dtype='int32')
    input_masks_ids = tf.keras.layers.Input(shape=(max_length,), name='attention_mask', dtype='int32')
    token_type_ids = tf.keras.layers.Input(shape=(max_length,), name='token_type_ids', dtype='int32')
    X = transformer_model({'input_ids': input_ids, 'attention_mask': input_masks_ids, 'token_type_ids': token_type_ids})[0]
    model = tf.keras.Model(inputs=[input_ids, input_masks_ids, token_type_ids], outputs=X)
    model.summary()
    model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
                  optimizer=tf.optimizers.Adam(learning_rate=0.00005), metrics=['accuracy'])
    return model

def get_metrices(true_values, pred_values):
    cm = confusion_matrix(true_values, pred_values)
    acc_score = accuracy_score(true_values, pred_values)
    f1 = f1_score(true_values, pred_values, average="binary")
    precision = precision_score(true_values, pred_values, average="binary")
    recall = recall_score(true_values, pred_values, average="binary")
    metrices = {'confusion_matrix': cm,
                'acc_score': acc_score,
                'f1': f1,
                'precision': precision,
                'recall': recall
                }
    for k, v in metrices.items():
        print(k, ':\n', v)
    return metrices

# get tokenizer
tokenizer = get_tokenizer(PRETRAINED_MODEL)

# train
prompt = ["Hello", "Hello", "Hello", "Hello"]
next_sentence = ["How are you?", "Pizza", "How are you?", "Pizza"]
train_labels = [0, 1, 0, 1]
train_labels = to_categorical(train_labels)
train_inputs = tokenize_nsp_data(prompt, next_sentence, max_length)
train_data_features = get_data_features(train_inputs, max_length)

# val
prompt = ["Hello", "Hello", "Hello", "Hello"]
next_sentence = ["How are you?", "Pizza", "How are you?", "Pizza"]
val_labels = [0, 1, 0, 1]
val_labels = to_categorical(val_labels)
val_inputs = tokenize_nsp_data(prompt, next_sentence, max_length)
val_data_features = get_data_features(val_inputs, max_length)

# get transformer model
transformer_model = get_transformer_model(PRETRAINED_MODEL)

# get keras model
model = get_keras_model(transformer_model)

callback_list = []
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=4, min_delta=0.005, verbose=1)
callback_list.append(early_stop)
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, epsilon=0.001)
callback_list.append(reduce_lr)
callback_list.append(checkpoint)

history = model.fit([train_data_features['input_ids'], train_data_features['attention_mask'],
                     train_data_features['token_type_ids']], np.array(train_labels), batch_size=2, epochs=3,
                    validation_data=([val_data_features['input_ids'], val_data_features['attention_mask'],
                                      val_data_features['token_type_ids']], np.array(val_labels)), verbose=1,
                    callbacks=callback_list)

model.layers[3].save_pretrained(model_path)  # need to save this and make sure i can get the hidden states

##  predict
# load model
transformer_model = get_transformer_model(model_path)
model = get_keras_model(transformer_model)
model.summary()
model.load_weights(os.path.join(model_path, model_fn))

# test
prompt = ["Hello", "Hello"]
next_sentence = ["How are you?", "Pizza"]
test_labels = [0, 1]
test_df = pd.DataFrame({'A': prompt, 'B': next_sentence, 'label': test_labels})
test_labels = to_categorical(val_labels)
test_inputs = tokenize_nsp_data(prompt, next_sentence, max_length)
test_data_features = get_data_features(test_inputs, max_length)

# predict
pred_test = model.predict([test_data_features['input_ids'], test_data_features['attention_mask'], test_data_features['token_type_ids']])
preds = tf.keras.activations.softmax(tf.convert_to_tensor(pred_test)).numpy()

true_test = test_df['label'].to_list()
pred_test = [1 if p[1] > 0.5 else 0 for p in preds]
test_df['pred_val'] = pred_test

metrices = get_metrices(true_test, pred_test)

I am also attaching a picture from the debugging mode in which I try (with no success) to view the hidden state. The problem is I am not able to see and save the transform model I trained and view the embeddings of the last hidden state. I tried converting the KerasTensor to numpy array but without success.
Screen Shot 2021-01-24 at 17.36.052138×1522 253 KB

Context:
stackoverflow.com

  Using tensorflow and TFBertForNextSentencePrediction to further train bert on a specific corpus 13

  python, tensorflow, keras, huggingface-transformers

  asked by

    ayalaall

  on 03:40PM - 24 Jan 21 UTC


Cuda out of memory during evaluation but training is fine

Question:
Hi,
I am finetuning a BARTForConditionalGeneration model. I am using Trainer from the library to train so I do not use anything fancy. I have 2 gpus I can even fit batch size 8 or 16 during training but after first epoch, I always receive Cuda Out of memory error. I find it strange because my evaluation batch size is 1.
Below is my code, which is very short actually.
import torch
import argparse
import os
import sys
import numpy as np
import torch.nn.functional as F
sys.path.append('..')
from transformers import T5ForConditionalGeneration, Trainer, TrainingArguments, BartForConditionalGeneration, FSMTForConditionalGeneration
from data_reader import GetDataAsPython
from sklearn.model_selection import train_test_split
from prepare_data import create_data, create_dataset, get_test_results, extract_warning_types
from transformers import T5Tokenizer, BartTokenizer, FSMTTokenizer
from datetime import datetime

parser = argparse.ArgumentParser()
parser.add_argument('-e', '--epochs', type=int, default=100)
parser.add_argument('-bs', '--batch-size', type=int, default=1)
parser.add_argument('-lr', '--learning-rate', type=float, default=1e-4)
parser.add_argument('-gcv', '--gradient-clip-val', type=float, default=0.0)
parser.add_argument('-wd', '--weight-decay', type=float, default=0.01)
parser.add_argument('-mn', '--model-name', type=str, choices=['t5-small', 't5-base', 't5-large', 'bart-base'], required=True)
args = parser.parse_args()

data = GetDataAsPython('../data_large2.json')
data_eslint = GetDataAsPython('../data_eslint.json')
data += data_eslint

all_warning_types = extract_warning_types(data)
all_warning_types = ['generator-star-spacing', 'no-array-constructor', 'no-extra-bind', 'no-debugger', 'no-extra-boolean-cast', 'no-extra-semi', 'no-useless-escape']

model_name = args.model_name
if 't5' in model_name:
    tokenizer = T5Tokenizer.from_pretrained(model_name)
elif 'bart' in model_name:
    tokenizer = BartTokenizer.from_pretrained('facebook/' + model_name)
else:
    raise "Unrecognized model"
tokenizer.add_tokens(['{', '}'])

now = datetime.now()
dt_string = now.strftime("%d-%m-%Y_%H-%M-%S")
model_directory = 't5global' + '_' + dt_string
model_directory = model_name + '_global_' + dt_string
os.system('mkdir ' + model_directory)
with open(model_directory + '/commandline_args.txt', 'w') as f:
    f.write('\n'.join(sys.argv[1:]))
tokenizer.save_pretrained(model_directory)

train_inputs, train_labels, val_inputs, val_labels, test_inputs, test_labels = create_data(data, all_warning_types, include_warning=True, model_name=model_name)
train_dataset = create_dataset(train_inputs, train_labels, tokenizer, pad_truncate=True)
val_dataset = create_dataset(val_inputs, val_labels, tokenizer, pad_truncate=True)
test_dataset = create_dataset(test_inputs, test_labels, tokenizer, pad_truncate=True)

training_args = TrainingArguments(
    output_dir=model_directory,
    num_train_epochs=args.epochs,
    per_device_train_batch_size=args.batch_size,
    per_device_eval_batch_size=1,
    warmup_steps=500,
    weight_decay=args.weight_decay,
    logging_dir=model_directory,
    logging_steps=100,
    do_eval=True,
    evaluation_strategy='epoch',
    learning_rate=args.learning_rate,
    load_best_model_at_end=True,
    metric_for_best_model='eval_loss',
    greater_is_better=False,
)

if 't5' in model_name:
    model = T5ForConditionalGeneration.from_pretrained(model_name, return_dict=False)
elif 'bart' in model_name:
    model = BartForConditionalGeneration.from_pretrained('facebook/' + model_name)
model.resize_token_embeddings(len(tokenizer))

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    optimizers=[torch.optim.Adam(params=model.parameters(), lr=args.learning_rate), None],
    tokenizer=tokenizer,
)

trainer.train()
trainer.save_model()
output = get_test_results(model, tokenizer, test_inputs, test_labels, False)
print(output)

output_file = open(model_name + 'allrules_results.txt', 'w+')
output_file.write(output)

and here is the stack trace
{'loss': 7.759439697265625, 'learning_rate': 2e-05, 'epoch': 0.28328611898017}

{‘loss’: 1.2010345458984375, ‘learning_rate’: 4e-05, ‘epoch’: 0.56657223796034}
{‘loss’: 0.3362786865234375, ‘learning_rate’: 6e-05, ‘epoch’: 0.8498583569405099}
3%|█████▎                                                                                | 353/10590 [02:50<1:22:07,  2.08it/sTraceback (most recent call last):██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉    | 40/41 [00:09<00:00,  3.72it/s]
File “transformers_global.py”, line 87, in
trainer.train()
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer.py”, line 792, in train
self._maybe_log_save_evalute(tr_loss, model, trial, epoch)
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer.py”, line 843, in _maybe_log_save_evalute
metrics = self.evaluate()
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer.py”, line 1251, in evaluate
output = self.prediction_loop(eval_dataloader, description=“Evaluation”)
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer.py”, line 1353, in prediction_loop
preds_host = logits if preds_host is None else nested_concat(preds_host, logits, dim=0)
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer_pt_utils.py”, line 47, in nested_concat
return type(tensors)(nested_concat(t, n, dim) for t, n in zip(tensors, new_tensors))
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer_pt_utils.py”, line 47, in
return type(tensors)(nested_concat(t, n, dim) for t, n in zip(tensors, new_tensors))
File “/home/berkay/model/lib/python3.8/site-packages/transformers/trainer_pt_utils.py”, line 49, in nested_concat
return torch.cat((tensors, new_tensors), dim=dim)
RuntimeError: CUDA out of memory. Tried to allocate 2.63 GiB (GPU 0; 10.76 GiB total capacity; 4.74 GiB already allocated; 2.53 GiB free; 7.27 GiB reserved in total by PyTorch)
3%|█████▎

Context: To avoid that, you need to add eval_accumulation_steps in your TrainingArguments. By default the Trainer accumulated all predictions on the host before sending them to the CPU (because it’s faster) but if you run OOM, fix that argument to a small value (for instance 20 or 10) to trigger the copy more frequently and free host memory.


Question:
This is an error with the MLM script (PyTorch) for attempting to pre-train BigBird on TPUs over XLA. The dataset in question is a custom dataset, and the model config and tokenizer has been initialized appropriately.
This is a continuation of this  unanswered 1 Forum post that faces the same error.
Command used to run the script:-
%%bash
python xla_spawn.py --num_cores=8 ./run_mlm.py --output_dir="./results" \
    --model_type="big_bird" \
    --config_name="./config" \
    --tokenizer_name="./tokenizer" \
    --train_file="./dataset.txt" \
    --validation_file="./val.txt" \
    --line_by_line="True" \
    --max_seq_length="16000" \
    --weight_decay="0.01" \
    --per_device_train_batch_size="1" \
    --per_device_eval_batch_size="1" \
    --learning_rate="3e-4" \
    --tpu_num_cores='8' \
    --warmup_steps="1000" \
    --overwrite_output_dir \
    --pad_to_max_length \
    --num_train_epochs="5" \
    --adam_beta1="0.9" \
    --adam_beta2="0.98" \
    --do_train \
    --do_eval \
    --logging_steps="50" \
    --evaluation_strategy="steps" \
    --eval_accumulation_steps='10' \
    --report_to="tensorboard" \
    --logging_dir='./logs' \
    --save_strategy="epoch" \
    --load_best_model_at_end='True' \
    --metric_for_best_model='validation' \
    --preprocessing_num_workers='15'

I am facing two errors to be precise,
Exception in device=TPU:0: Default process group has not been initialized, please make sure to call init_process_group.
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/transformers/training_args.py", line 1006, in main_process_first
    yield
  File "/content/run_mlm.py", line 393, in main
    desc="Running tokenizer on dataset line_by_line",
  File "/usr/local/lib/python3.7/dist-packages/datasets/dataset_dict.py", line 489, in map
    for k, dataset in self.items()
  File "/usr/local/lib/python3.7/dist-packages/datasets/dataset_dict.py", line 489, in <dictcomp>
    for k, dataset in self.items()
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 1664, in map
    for rank in range(num_proc)
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 1664, in <listcomp>
    for rank in range(num_proc)
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 2664, in shard
    writer_batch_size=writer_batch_size,
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 186, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/datasets/fingerprint.py", line 397, in wrapper
    out = func(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 2254, in select
    return self._new_dataset_with_indices(indices_buffer=buf_writer.getvalue(), fingerprint=new_fingerprint)
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 2170, in _new_dataset_with_indices
    fingerprint=fingerprint,
  File "/usr/local/lib/python3.7/dist-packages/datasets/arrow_dataset.py", line 297, in __init__
    self._indices.column(0)[0].type
  File "pyarrow/table.pxi", line 162, in pyarrow.lib.ChunkedArray.__getitem__
  File "pyarrow/array.pxi", line 549, in pyarrow.lib._normalize_index
IndexError: index out of bounds

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/torch_xla/distributed/xla_multiprocessing.py", line 329, in _mp_start_fn
    _start_fn(index, pf_cfg, fn, args)
  File "/usr/local/lib/python3.7/dist-packages/torch_xla/distributed/xla_multiprocessing.py", line 323, in _start_fn
    fn(gindex, *args)
  File "/content/run_mlm.py", line 529, in _mp_fn
    main()
  File "/content/run_mlm.py", line 393, in main
    desc="Running tokenizer on dataset line_by_line",
  File "/usr/lib/python3.7/contextlib.py", line 130, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/lib/python3.7/dist-packages/transformers/training_args.py", line 1011, in main_process_first
    torch.distributed.barrier()
  File "/usr/local/lib/python3.7/dist-packages/torch/distributed/distributed_c10d.py", line 2523, in barrier
    default_pg = _get_default_group()
  File "/usr/local/lib/python3.7/dist-packages/torch/distributed/distributed_c10d.py", line 358, in _get_default_group
    raise RuntimeError("Default process group has not been initialized, "
RuntimeError: Default process group has not been initialized, please make sure to call init_process_group.

I haven’t modified the script to call the init_process_group yet, focusing on the earlier error of index out of bounds. Clearly, the problem is arising from my own dataset - which was working before however. Interestingly, we get it when its in the tokenizing stage.
At some point, when constructing the arrow dataset its failing. I have no idea about Apache Arrow, so I can’t debug further
Can anyone give me some guidance on where should I start to investigate the error and some possible leads as to the origin?

Context: Any ideas anyone?


How to print a few examples at the beginning of training when using Trainer?

Question: I would like to see a few examples to manually verify the input and output. Previously in an older version of transformers there is a functionality of doing that. I am wondering whether there is something similar for trianer?

Context: This is not something related to the Trainer, you just have to print some elements of your dataset. The functionality is implemented in all examples, see for instance this one 11


Using Trainer class with T5 - what is returned in EvalPrediction dict?

Question:
Hi,
I am trying to finetune T5 using the Trainer class. I understand that Trainer doesn’t work out-of-the-box for seq2seq tasks and saw @patrickvonplaten’s https://github.com/huggingface/transformers/pull/5840 9 which extends the Trainer class to work for Bert2Bert. Is my understanding correct that the Trainer class appropriately handles training for seq2seq models since the loss is calculated by the model itself, and that the only problem is when returning EvalPredictions for calculating and logging custom validation metrics?
If so then I would really appreciate if someone can help me to understand what’s being returned in the EvalPrediction dict for T5, it seems like EvalPrediction.predictions is of size batch_size * max_output_len * model_size (31218), is this the generated prediction in embedding form? If so what is the best way to convert this to prediction ids? I tried naively calling model.lm_head() on it but that didn’t seem to be the correct approach. @valhalla perhaps you can weigh in, I also took a look at your notebook finetuning T5 with Pytorch Lightning but would really like to use the HF Trainer class.
Thanks for all help.

Context:
Hi @melody-ju
Not sure what you mean by  "T5 doesn’t work out-of-the-box for seq2seq tasks ". T5 is a seq2seq model and it does work for seq2seq tasks.
You can use Trainer for seq2seq tasks as it is. Patrick’s PR extends it so that generative metrics can be calculated (ROUGE, BLUE etc), it should be okay if you calculate them after training the training is finished.
To use Trainer for T5, the dataset or collator (if you are using one) should at least return
input_ids, attention_mask and labels (set pad tokens to -100 in labels). The rest will be handled by Trainer
This notebook 33 uses Trainer for fine-tuning T5.
Few things to note about  that notebook,
I wrote it before v3.0.0, few things have changed after that

DatCollator is not a class anymore, so you won’t need to inherit from DataCollator when creating T2TDataCollator. Also collate_batch should be renamed to __call__.

lm_lables is now deprecated, use labels instead.
No need to manually add </s> anymore, the tokenizer  now does that itself.

Also you can use the prepare_seq2seq_batch method on toknizer which can take the source and target text and returns input_ids,  attention_mask and labels.
You can also use finetune.py script from here 11 to finetune T5 and other seq2seq models. It’s using PL, and there;s WIP version of Seq2SeqTrainer in this PR 5


TypeError: forward() got an unexpected keyword argument ‘start_positions’

Question:
Hello everyone. I already have post a question about fine-tuning bert-base-italian-cased on SQuAD-it dateset. Waiting for an answer I tried another solution, following the Question Answerinf tutorial on SQuAS 2.0 in the transformers docs on HuggingFace.
My data are taken from SQuAD-it. I followed this way:
import json
from pathlib import Path

def read_dataset(path):
    path = Path(path)
    with open(path, 'rb') as f:
        squad_dict = json.load(f)

    contexts = []
    questions = []
    answers = []
    for group in squad_dict['data']:
        for passage in group['paragraphs']:
            context = passage['context']
            for qa in passage['qas']:
                question = qa['question']
                for answer in qa['answers']:
                    contexts.append(context)
                    questions.append(question)
                    answers.append(answer)

    return contexts, questions, answers

train_contexts, train_questions, train_answers = read_dataset('SQuAD_it-train.json')

val_contexts = []
val_questions = []
val_answers = []

while len(val_answers) != 5831:
  value = train_contexts.pop()
  val_contexts.append(value)
  value = train_questions.pop()
  val_questions.append(value)
  value = train_answers.pop()
  val_answers.append(value)

def add_end_idx(answers, contexts):
    for answer, context in zip(answers, contexts):
        gold_text = answer['text']
        start_idx = answer['answer_start']
        end_idx = start_idx + len(gold_text)

        # sometimes squad answers are off by a character or two – fix this
#        if context[start_idx:end_idx] == gold_text:
#            answer['answer_end'] = end_idx
        if context[start_idx-1:end_idx-1] == gold_text:
            answer['answer_start'] = start_idx - 1
            answer['answer_end'] = end_idx - 1     # When the gold label is off by one character
        elif context[start_idx-2:end_idx-2] == gold_text:
            answer['answer_start'] = start_idx - 2
            answer['answer_end'] = end_idx - 2     # When the gold label is off by two characters
        elif context[start_idx-1:end_idx-2] == gold_text:
            answer['answer_start'] = start_idx - 1
            answer['answer_end'] = end_idx - 2
        elif context[start_idx-2:end_idx-1] == gold_text:
            answer['answer_start'] = start_idx - 2
            answer['answer_end'] = end_idx - 1
        elif context[start_idx-3:end_idx-3] == gold_text:
            answer['answer_start'] = start_idx - 3
            answer['answer_end'] = end_idx - 3
        elif context[start_idx-2:end_idx-3] == gold_text:
            answer['answer_start'] = start_idx - 2
            answer['answer_end'] = end_idx - 3
        elif context[start_idx-3:end_idx-2] == gold_text:
            answer['answer_start'] = start_idx - 3
            answer['answer_end'] = end_idx - 2
        else:
          answer['answer_end'] = end_idx
        if answer['answer_start'] < 0:
            answer['answer_start'] =+ 1
            answer['answer_end'] =+ 1

add_end_idx(train_answers, train_contexts)
add_end_idx(val_answers, val_contexts)

from transformers import DistilBertTokenizerFast
tokenizer = DistilBertTokenizerFast.from_pretrained('dbmdz/bert-base-italian-cased')

train_encodings = tokenizer(train_contexts, train_questions, truncation=True, padding=True)
val_encodings = tokenizer(val_contexts, val_questions, truncation=True, padding=True)

from transformers import AutoModel,
model_name = "dbmdz/bert-base-italian-cased"
model = AutoModel.from_pretrained(model_name)

def add_token_positions(encodings, answers):
    start_positions = []
    end_positions = []
    for i in range(len(answers)):
        start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))
        end_positions.append(encodings.char_to_token(i, answers[i]['answer_end'] - 1))

        # if start position is None, the answer passage has been truncated
        if start_positions[-1] is None:
            start_positions[-1] = tokenizer.model_max_length
        if end_positions[-1] is None:
            end_positions[-1] = tokenizer.model_max_length

    encodings.update({'start_positions': start_positions, 'end_positions': end_positions})

add_token_positions(train_encodings, train_answers)
add_token_positions(val_encodings, val_answers)

Then I created the datasets:
import torch

class SquadDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}

    def __len__(self):
        return len(self.encodings.input_ids)

train_dataset = SquadDataset(train_encodings)
val_dataset = SquadDataset(val_encodings)

And finally I tried to train:
from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir='./results',          # output directory
    num_train_epochs=3,              # total number of training epochs
    per_device_train_batch_size=16,  # batch size per device during training
    per_device_eval_batch_size=64,   # batch size for evaluation
    warmup_steps=500,                # number of warmup steps for learning rate scheduler
    weight_decay=0.01,               # strength of weight decay
    logging_dir='./logs',            # directory for storing logs
    logging_steps=10,
    label_names = ["start_positions", "end_positions"]
)

trainer = Trainer(
    model=model,                         # the instantiated 🤗 Transformers model to be trained
    args=training_args,                  # training arguments, defined above
    train_dataset=train_dataset,         # training dataset
    eval_dataset=val_dataset             # evaluation dataset
)

trainer.train()

But it raises me this error:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-171-8794092ae722> in <module>()
     20 )
     21
---> 22 trainer.train()

3 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

TypeError: forward() got an unexpected keyword argument 'start_positions'

I’ve seen this has already an issue but in none topic I’ve found a solution. Thank you in advance.

Context: Like in the other subject, we need to know how you created your model. It looks like that model is not happy with start_positions so it’s very likely not a question-answering model.


“I am gay.” sentence is classified as NEGATIVE with score 0.99

Question:
This code gives the following output:
from transformers import pipeline
classifier = pipeline(“sentiment-analysis”)
#classifier(“I’ve been waiting for a HuggingFace course my whole life.”)
classifier(“I am gay.”)
output:
[{‘label’: ‘NEGATIVE’, ‘score’: 0.9939725399017334}]
“I am gay.” sentence is classified as NEGATIVE with score of 0.99.
Should the dataset be modified or processed again?

Context: That’s part of the warnings about pretrained models included here 5. Every fine-tuned versions gets the bias of the original pretrained model.


ValueError: Mixed precision training with AMP or APEX (`–fp16`) and FP16 evaluation can only be used on CUDA devices

Question:
I am trying to tune Wav2Vec2 Model with a dataset on my local device using my CPU (I don’t have a GPU or Google Colab pro), I am using this 4 as my reference. When I try to execute
from transformers import TrainingArguments

training_args = TrainingArguments(
  # output_dir="/content/gdrive/MyDrive/wav2vec2-base-timit-demo",
  output_dir="./wav2vec2-medical",
  group_by_length=True,
  per_device_train_batch_size=32,
  evaluation_strategy="steps",
  num_train_epochs=30,
  fp16=True,
  save_steps=500,
  eval_steps=500,
  logging_steps=500,
  learning_rate=1e-4,
  weight_decay=0.005,
  warmup_steps=1000,
  save_total_limit=2,
)

I am getting following error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-26-f9014a6221db> in <module>
      1 from transformers import TrainingArguments
      2
----> 3 training_args = TrainingArguments(
      4   # output_dir="/content/gdrive/MyDrive/wav2vec2-base-timit-demo",
      5   output_dir="./wav2vec2-medical",

~/Library/Python/3.8/lib/python/site-packages/transformers/training_args.py in __init__(self, output_dir, overwrite_output_dir, do_train, do_eval, do_predict, evaluation_strategy, prediction_loss_only, per_device_train_batch_size, per_device_eval_batch_size, per_gpu_train_batch_size, per_gpu_eval_batch_size, gradient_accumulation_steps, eval_accumulation_steps, learning_rate, weight_decay, adam_beta1, adam_beta2, adam_epsilon, max_grad_norm, num_train_epochs, max_steps, lr_scheduler_type, warmup_ratio, warmup_steps, logging_dir, logging_strategy, logging_first_step, logging_steps, save_strategy, save_steps, save_total_limit, no_cuda, seed, fp16, fp16_opt_level, fp16_backend, fp16_full_eval, local_rank, tpu_num_cores, tpu_metrics_debug, debug, dataloader_drop_last, eval_steps, dataloader_num_workers, past_index, run_name, disable_tqdm, remove_unused_columns, label_names, load_best_model_at_end, metric_for_best_model, greater_is_better, ignore_data_skip, sharded_ddp, deepspeed, label_smoothing_factor, adafactor, group_by_length, length_column_name, report_to, ddp_find_unused_parameters, dataloader_pin_memory, skip_memory_metrics, use_legacy_prediction_loop, push_to_hub, resume_from_checkpoint, mp_parameters)

~/Library/Python/3.8/lib/python/site-packages/transformers/training_args.py in __post_init__(self)
    609
    610         if is_torch_available() and self.device.type != "cuda" and (self.fp16 or self.fp16_full_eval):
--> 611             raise ValueError(
    612                 "Mixed precision training with AMP or APEX (`--fp16`) and FP16 evaluation can only be used on CUDA devices."
    613             )

ValueError: Mixed precision training with AMP or APEX (`--fp16`) and FP16 evaluation can only be used on CUDA devices.

I understand that the error is because I am not using a GPU, as mixed precision can not be carried out without a GPU. I want to run it on my CPU, how can I resolve the error

Context: You should remove fp16=True then.


How to train the Translation Language Modeling (TLM) with transformers/examples/language-modeling/run_mlm.py?

Question:
Hello, I have a question to ask for your help. I want to train the Translation Language Modeling (TLM) in XLM (Paper: Cross-lingual Language Model Pretraining). The translation language modeling (TLM) is very similar to the  Masked Language Modeling (MLM), which only shows the difference in the form of input data. If I want to use the run_mlm.py file to achieve the effect of training the translation language modeling (TLM), can I just modify the composition of training data without modifying the source code of the transformers/examples/language-modeling/run_mlm.py file? Is this feasible?
For example, for the masked language modeling (MLM), one row of my training data is a language, as shown below:
( Row 1 ) polonium 's isotopes tend to decay with alpha or beta decay ( en ) .
( Row 2 ) 231 and penetrated the armour of the Panzer IV behind it ( en ) .
( Row 3 ) die Isotope von Polonium neigen dazu , mit dem Alpha- oder Beta-Zerfall zu zerfallen ( de ) .
( Row 4 ) 231 und durchbrach die Rüstung des Panzers IV hinter ihm ( de ) .
…
For the translation language modeling (TLM), my training data is a combination of two parallel corpora (It is to splice the above data in pairs. The separator is [/s] [/s].), as shown below:
( Row 1 ) polonium 's isotopes tend to decay with alpha or beta decay ( en ) . [/s] [/s] die Isotope von Polonium neigen dazu , mit dem Alpha- oder Beta-Zerfall zu zerfallen ( de ) .
( Row 2 ) 231 and penetrated the armour of the Panzer IV behind it ( en ) . [/s] [/s] 231 und durchbrach die Rüstung des Panzers IV hinter ihm ( de ) .
…
If I only modify the training data into a combination of two parallel corpora before executing the transformers/examples/language-modeling/run_mlm.py file, can I achieve the effect of training the translation language modeling (TLM)?
Looking forward to your help, thank you very much!

Context: I have same confuse. Could tell me if you solve this issue？


Question:
This is in continuation of my previous issue filed in this forum (here 2).
I wanted to try and run the run_mlm_flax script with a Colab TPU v2, but I am facing an error:-
Module Name: <module 'run_mlm_flax' from '/content/run_mlm_flax.py'>

WARNING:root:TPU has started up successfully with version pytorch-1.9
Traceback (most recent call last):
  File "xla_spawn.py", line 87, in <module>
    main()
  File "xla_spawn.py", line 83, in main
    xmp.spawn(mod._mp_fn, args=(), nprocs=args.num_cores)
AttributeError: module 'run_mlm_flax' has no attribute '_mp_fn'

To my understanding, the script for spawning processes in each TPU core required a Huggingface Trainer object to work with. However, we apparently do not have a Trainer script for MLM/CLM for Bigbird. Compounding the difficulties is the whole Flax-noTrainer-TPU combination which I do not think BigBird has been tested out fully.
Is there any possible approach to run this on TPUs?

Context:
Hey, I think you are mixing flax & pytorch scripts. how can you get pytorch error in flax script otherwise?
FlaxBigBird works perfectly on cloud TPU. You can refer this script 4 also.
Also Currently Trainer is for only PyTorch & Tensorflow.


Question:
Hello, did someone already try to get the timestamps of the words while decoding the audio with wav2vec2 model?
Thank you!

Context: I have created some hacky code to do so and have posted it in this huggingface github issue 14. You can find it here: Getting time offsets of beginning and end of each word in Wav2Vec2 · Issue #11307 · huggingface/transformers · GitHub 56


BUG Confirmation: BigBirdLM not able to use Flax

Question:
On Google Colab, trying to use FlaxBigBirdForMaskedLM shows that Flax is not installed.
from transformers import BigBirdConfig

config = BigBirdConfig(
    vocab_size=40000,
    hidden_size = 768,
    max_position_embeddings=16000,
    num_attention_heads=4,                   #6
    num_hidden_layers=4,                    #6
)

from transformers import FlaxBigBirdForMaskedLM
model = FlaxBigBirdForMaskedLM(config=config)

It may be due to some other issue, But I am consistenly getting this error across all runtimes (target runtime is TPU).
ImportError:
FlaxBigBirdForMaskedLM requires the FLAX library but it was not found in your environment. Checkout the instructions on the
installation page: https://github.com/google/flax and follow the ones that match your environment.

Flax is indeed installed, via !pip install -q transformers[flax].
Does this seem like a genuine bug? The problem is that the backend Flax is apparently not accessible by the method, while I can easily import flax and other utilities.

Context: cc @vasudevgupta who i believe is the one working on bigbird and flax


How to save RoBERTA sequence classifier model

Question:
After finishing the training phase, I want to save the RoBERTa classifier model for later evaluation purpose.

I am getting this error.
AttributeError                            Traceback (most recent call last)
 in
1 out_model = “src_classifier_final_ICWSM_10e”
----> 2 src_classifier.save_model(f’models/{out_model}_final’)
~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py in getattr(self, name)
945             if name in modules:
946                 return modules[name]
 → 947         raise AttributeError("’{}’ object has no attribute ‘{}’".format(
948             type(self).name, name))
949
AttributeError: ‘RobertaForSequenceClassification’ object has no attribute ‘save_model’

Context: I’m not sure where you have gotten that save_model method but Transformers model don’t have it. They use save_pretrained.


Question:
Hello. I am trying to greedy_search with T5ModelForConditionalGeneration.
However, it seems I have to provide input_ids as both the input_ids parameter and as part of model_kwargs parameter of the greedy_search.
The input_ids argument of greedy_search acts as the initial decoded state, while input_ids that is supposed to appear in model_kwargs  is passed to self (T5) for inference.
I do not see a way to pass two input_ids argument to the greedy_search function. Right now the function complains that I don’t pass input_ids to its internal T5.
How would I go about using the greedy_search function fot T5?

Context: Solved by passing encoder_outputs to greedy_seach.


Tutorials for using Colab TPUs with Huggingface Transformers?

Question:
I looking for an easy-to-follow tutorial for using Huggingface Transformer models (e.g. BERT) in PyTorch on Google Colab with TPUs. I found guides about XLA, but they are largely centered around TensorFlow.
Any help would be appreciated.

Context:
There are a few contributed notebooks which might help you here: https://github.com/huggingface/transformers/tree/master/notebooks 623
For instance this one by @valhalla is about training T5 on TPU in PyTorch: https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb#scrollTo=QLGiFCDqvuil 560


Question:
Hi @patrickvonplaten,
Thanks for offering to help me!
For anyone else reading, this is a continuation of the discussion on huggingface/transformers issue 12137. In short, despite following the official wav2vec2 English ASR guide 2, my model cannot converge on my dataset of single English word, 1 second long audio clips.
When trying out the guide’s Colab Notebook on the TIMIT dataset, however, the model converges just fine which is perplexing to me.
Here is my training notebook 9.

Context:
You probably need to train a bit longer. Did you train the full 30 epochs? Or did you stop at 2?
The wav2vec2 embeddings only learn the representations of speech, it does not know how to output characters. The finetuning stage learns to use the embeddings to output characters.
The usual finetuning training behavior looks sth like:

Beginning: Output random chars
Early: Output nothing - empty strings - looks like you are here?
After a while: Starts to spit out more relevant chars


How to set up DistilBertModel to use a bach_size?

Question:
Goal: I want to get the [CLS] values, but I am getting an error when I call DistilBertModel.
My code:
import transformers as ppb
m, t, p = (ppb.DistilBertModel, ppb.DistilBertTokenizerFast, 'distilbert-base-uncased')

tokenizer = t.from_pretrained(pretrained_weights, cache_dir=<path>)
model = m.from_pretrained(pretrained_weights, from_tf=True, cache_dir=<path>)
# [beginning of EDIT]
def my_encode(tokenizer, texts, max_length=MAX_LENGTH):
        inputs = tokenizer.batch_encode_plus(texts,
                                             max_length=max_length,
                                             padding='longest',
                                             truncation=True,
                                             return_attention_mask=True,
                                             return_token_type_ids=False,
                                             return_tensors="pt"
                                             )
    return inputs

tokenizer_output = my_encode(tokenizer, pandas_df['raw_text'].tolist())
# [end of EDIT]

I am getting error when I call the model with ‘tokenizer_output’, which is a ‘transformers.tokenization_utils_base.BatchEncoding’:
result = model(**tokenizer_output)
The error is:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<timed exec> in <module>

~/miniconda3/envs/x/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~/miniconda3/envs/x/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)
    485             output_attentions=output_attentions,
    486             output_hidden_states=output_hidden_states,
--> 487             return_dict=return_dict,
    488         )
    489

~/miniconda3/envs/x/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~/miniconda3/envs/x/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, x, attn_mask, head_mask, output_attentions, output_hidden_states, return_dict)
    305
    306             layer_outputs = layer_module(
--> 307                 x=hidden_state, attn_mask=attn_mask, head_mask=head_mask[i], output_attentions=output_attentions
    308             )
    309             hidden_state = layer_outputs[-1]

~/miniconda3/envs/x/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~/miniconda3/envs/x/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, x, attn_mask, head_mask, output_attentions)
    262
    263         # Feed Forward Network
--> 264         ffn_output = self.ffn(sa_output)  # (bs, seq_length, dim)
    265         ffn_output = self.output_layer_norm(ffn_output + sa_output)  # (bs, seq_length, dim)
    266

~/miniconda3/envs/x/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~/miniconda3/envs/x/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, input)
    213
    214     def forward(self, input):
--> 215         return apply_chunking_to_forward(self.ff_chunk, self.chunk_size_feed_forward, self.seq_len_dim, input)
    216
    217     def ff_chunk(self, input):

~/miniconda3/envs/x/lib/python3.7/site-packages/transformers/modeling_utils.py in apply_chunking_to_forward(forward_fn, chunk_size, chunk_dim, *input_tensors)
   1815         return torch.cat(output_chunks, dim=chunk_dim)
   1816
-> 1817     return forward_fn(*input_tensors)

~/miniconda3/envs/x/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in ff_chunk(self, input)
    217     def ff_chunk(self, input):
    218         x = self.lin1(input)
--> 219         x = self.activation(x)
    220         x = self.lin2(x)
    221         x = self.dropout(x)

~/miniconda3/envs/x/lib/python3.7/site-packages/torch/nn/functional.py in gelu(input)
   1457     if has_torch_function_unary(input):
   1458         return handle_torch_function(gelu, (input,), input)
-> 1459     return torch._C._nn.gelu(input)
   1460
   1461

RuntimeError: [enforce fail at CPUAllocator.cpp:67] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 2826240000 bytes. Error code 12 (Cannot allocate memory)

I believe if call model using batch would solve my problem. But how I can I use batch_size here?
result = model(**tokenizer_output)
Is there another way to get the [CLS] (word representation)?
Thanks in advance!

Context: You did not share how you are building your tokenizer_output, so it’s hard to help you.


Question: I wanted to know how we can do source attribution with current implementation of CTRL as shown here 3.

Context: @patrickvonplaten  Can you suggest something ?


Question:
Feature request
How can I continue training on a Fine-tuned Model?
I have a fine tuned model from OpenSLR data. And I want to continue training on an model as I continue to gain transcribed audio data over time. Can I do like making the fine-tuned model as a checkpoint?

Motivation
I am aiming to make a model for Nepali Language. I have a way to collect data over time and it is continuous. So, I want to find a way I can train the model continuously as I gain data over time

Context: In PyTorch, you can further train a model just by putting it in training mode (model.train()), and then train as usual. This will update the parameters of the model.


How to Improve inference time of facebook/mbart many to many model?

Question: If we tried to run translation service on facebook mbart many to many on cpu it take 9 secs to translate, how do we reduce the inference time further…

Context:
Hi @Vimal0703, one idea could be to try quantizing the model’s weights to a lower precision datatype. See e.g. step 2 in this guide: Dynamic Quantization — PyTorch Tutorials 1.7.1 documentation 16
This usually gives you a 2-3x reduction in latency and model size


How do we quantize facebook / mbart-large-50-one-to-many-mmt to ONNX runtime

Question: I want to know where to find the mbart-large-50-one-to-many-mmt model so that I can download and convert to onnx model and please do suggest a way to convert this model to onnx runtime.

Context:
Have you seen these:

      huggingface.co

Hugging Face – On a mission to solve NLP, one commit at a time. 5

https://huggingface.co/transformers/serialization.html 10

      github.com

    huggingface/transformers/blob/master/notebooks/04-onnx-export.ipynb 16
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jBasof3bv1LB"
   },
   "source": [
    "
How to export 🤗 Transformers Models to ONNX ?
"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[ONNX](http://onnx.ai/) is open format for machine learning models. It allows to save your neural network's computation graph in a framework agnostic way, which might be particulary helpful when deploying deep learning models.\n",
    "\n",
    "Indeed, businesses might have other requirements _(languages, hardware, ...)_ for which the training framework might not be the best suited in inference scenarios. In that context, having a representation of the actual computation graph that can be shared accross various business units and logics across an organization might be a desirable component.\n",
    "\n",

  This file has been truncated. show original


gpt-neo-2.7B isn’t working with pipleline

Question:
I’m getting a basic error when I try to access GTP-NEO-2.7B via a pipeline. Working from my local machine, I go through the exact stages on the relevant model card page, namely:
from transformers import pipeline
generator = pipeline('text-generation', model='EleutherAI/gpt-neo-2.7B')

However, when I do so I get the following error:
OSError: Can't load config for 'EleutherAI/gpt-neo-2.7B'. Make sure that:

- 'EleutherAI/gpt-neo-2.7B' is a correct model identifier listed on 'https://huggingface.co/models'

- or 'EleutherAI/gpt-neo-2.7B' is the correct path to a directory containing a config.json file

Can anyone advise what’s happening here? Thanks.

Context: I think I had the same problem.  I went to the Huggingface site and copied the identifier from there and pasted it into my notebook.  Then it worked.  Even though the spelling seemed correct, it fixed the problem.  I’m not sure why.


Cannot convert mbart from fairseq to huggingface using the script in the repo

Question:
I am using this converter script in the transformers repo 4 to convert the official fairseq bart to huggingface. The command looks like:
python convert_mbart_original_checkpoint_to_pytorch.py mbart.cc25.v2/model.pt ./temp/

which returns an error of
	Unexpected key(s) in state_dict: "encoder.layers.0.layer_norms.0.weight", "encoder.layers.0.layer_norms.0.bias", "encoder.layers.0.layer_norms.1.weight", "encoder.layers.0.layer_norms.1.bias", "encoder.layers.0.self_attn.in_proj_weight", "encoder.layers.0.self_attn.in_proj_bias", "encoder.layers.1.layer_norms.0.weight", "encoder.layers.1.layer_norms.0.bias", "encoder.layers.1.layer_norms.1.weight", "encoder.layers.1.layer_norms.1.bias", "encoder.layers.1.self_attn.in_proj_weight", "encoder.layers.1.self_attn.in_proj_bias", "encoder.layers.2.layer_norms.0.weight", "encoder.layers.2.layer_norms.0.bias", "encoder.layers.2.layer_norms.1.weight", "encoder.layers.2.layer_norms.1.bias", "encoder.layers.2.self_attn.in_proj_weight", "encoder.layers.2.self_attn.in_proj_bias", "encoder.layers.3.layer_norms.0.weight", "encoder.layers.3.layer_norms.0.bias", "encoder.layers.3.layer_norms.1.weight", "encoder.layers.3.layer_norms.1.bias", "encoder.layers.3.self_attn.in_proj_weight", "encoder.layers.3.self_attn.in_proj_bias", "encoder.layers.4.layer_norms.0.weight", "encoder.layers.4.layer_norms.0.bias", "encoder.layers.4.layer_norms.1.weight", "encoder.layers.4.layer_norms.1.bias", "encoder.layers.4.self_attn.in_proj_weight", "encoder.layers.4.self_attn.in_proj_bias", "encoder.layers.5.layer_norms.0.weight", "encoder.layers.5.layer_norms.0.bias", "encoder.layers.5.layer_norms.1.weight", "encoder.layers.5.layer_norms.1.bias", "encoder.layers.5.self_attn.in_proj_weight", "encoder.layers.5.self_attn.in_proj_bias", "encoder.layers.6.layer_norms.0.weight", "encoder.layers.6.layer_norms.0.bias", "encoder.layers.6.layer_norms.1.weight", "encoder.layers.6.layer_norms.1.bias", "encoder.layers.6.self_attn.in_proj_weight", "encoder.layers.6.self_attn.in_proj_bias", "encoder.layers.7.layer_norms.0.weight", "encoder.layers.7.layer_norms.0.bias", "encoder.layers.7.layer_norms.1.weight", "encoder.layers.7.layer_norms.1.bias", "encoder.layers.7.self_attn.in_proj_weight", "encoder.layers.7.self_attn.in_proj_bias", "encoder.layers.8.layer_norms.0.weight", "encoder.layers.8.layer_norms.0.bias", "encoder.layers.8.layer_norms.1.weight", "encoder.layers.8.layer_norms.1.bias", "encoder.layers.8.self_attn.in_proj_weight", "encoder.layers.8.self_attn.in_proj_bias", "encoder.layers.9.layer_norms.0.weight", "encoder.layers.9.layer_norms.0.bias", "encoder.layers.9.layer_norms.1.weight", "encoder.layers.9.layer_norms.1.bias", "encoder.layers.9.self_attn.in_proj_weight", "encoder.layers.9.self_attn.in_proj_bias", "encoder.layers.10.layer_norms.0.weight", "encoder.layers.10.layer_norms.0.bias", "encoder.layers.10.layer_norms.1.weight", "encoder.layers.10.layer_norms.1.bias", "encoder.layers.10.self_attn.in_proj_weight", "encoder.layers.10.self_attn.in_proj_bias", "encoder.layers.11.layer_norms.0.weight", "encoder.layers.11.layer_norms.0.bias", "encoder.layers.11.layer_norms.1.weight", "encoder.layers.11.layer_norms.1.bias", "encoder.layers.11.self_attn.in_proj_weight", "encoder.layers.11.self_attn.in_proj_bias", "decoder.layers.0.self_attn.in_proj_weight", "decoder.layers.0.self_attn.in_proj_bias", "decoder.layers.0.encoder_attn.in_proj_weight", "decoder.layers.0.encoder_attn.in_proj_bias", "decoder.layers.1.self_attn.in_proj_weight", "decoder.layers.1.self_attn.in_proj_bias", "decoder.layers.1.encoder_attn.in_proj_weight", "decoder.layers.1.encoder_attn.in_proj_bias", "decoder.layers.2.self_attn.in_proj_weight", "decoder.layers.2.self_attn.in_proj_bias", "decoder.layers.2.encoder_attn.in_proj_weight", "decoder.layers.2.encoder_attn.in_proj_bias", "decoder.layers.3.self_attn.in_proj_weight", "decoder.layers.3.self_attn.in_proj_bias", "decoder.layers.3.encoder_attn.in_proj_weight", "decoder.layers.3.encoder_attn.in_proj_bias", "decoder.layers.4.self_attn.in_proj_weight", "decoder.layers.4.self_attn.in_proj_bias", "decoder.layers.4.encoder_attn.in_proj_weight", "decoder.layers.4.encoder_attn.in_proj_bias", "decoder.layers.5.self_attn.in_proj_weight", "decoder.layers.5.self_attn.in_proj_bias", "decoder.layers.5.encoder_attn.in_proj_weight", "decoder.layers.5.encoder_attn.in_proj_bias", "decoder.layers.6.self_attn.in_proj_weight", "decoder.layers.6.self_attn.in_proj_bias", "decoder.layers.6.encoder_attn.in_proj_weight", "decoder.layers.6.encoder_attn.in_proj_bias", "decoder.layers.7.self_attn.in_proj_weight", "decoder.layers.7.self_attn.in_proj_bias", "decoder.layers.7.encoder_attn.in_proj_weight", "decoder.layers.7.encoder_attn.in_proj_bias", "decoder.layers.8.self_attn.in_proj_weight", "decoder.layers.8.self_attn.in_proj_bias", "decoder.layers.8.encoder_attn.in_proj_weight", "decoder.layers.8.encoder_attn.in_proj_bias", "decoder.layers.9.self_attn.in_proj_weight", "decoder.layers.9.self_attn.in_proj_bias", "decoder.layers.9.encoder_attn.in_proj_weight", "decoder.layers.9.encoder_attn.in_proj_bias", "decoder.layers.10.self_attn.in_proj_weight", "decoder.layers.10.self_attn.in_proj_bias", "decoder.layers.10.encoder_attn.in_proj_weight", "decoder.layers.10.encoder_attn.in_proj_bias", "decoder.layers.11.self_attn.in_proj_weight", "decoder.layers.11.self_attn.in_proj_bias", "decoder.layers.11.encoder_attn.in_proj_weight", "decoder.layers.11.encoder_attn.in_proj_bias".

Am I missing anything here? Thanks!

Context: still needs help on this…


`Trainer.predict` takes twice as long as progress bar shows

Question:
Am using Trainer.predict but have noticed that it’s actually taking twice as long as displayed by the progress bar.
print('Predicting on test samples...')
t0 = time.time()
predictions, label_ids, _ = trainer.predict(tokenized_datasets['test'])
print(f'completed in {(time.time() - t0) / 60:.2f} mins.')
print('Argmaxing...')
t0 = time.time()
predictions = predictions.argmax(axis=2)
print(f'completed in {(time.time() - t0) / 60:.2f} mins.')

trainer_predict_timing1040×226 11.9 KB

During the period shown by the progress bar, the GPU is used, but it’s not used after the progress bar has completed.  Different sizes of tokenized_datasets['test'] has been tried, and it appears that the trend is that it takes twice as long regardless of the size.
Is this normal, or expected?

Context: It depends on the Trainer you are using, which you did not share. The progress bar only represents the evaluation loop (goign through all the batches of your evaluation set and getting the predictions). After this is done, the Trainer computes metrics if you gave it a compute metrics function, which could take some time, and there can also be some additional post-processing, depending on your task.


Is there a list of MLM corruption strategies?

Question:
Ideally, looking for a list of such corruption strategies that include an ELI5 description, pictures, and sample implementation(s).
Thanks -wg

Context:
Hi,
Yes, the T5 paper 2 explored different corruption strategies. See table 3 on page 21.


Question: When I run the GPU (google colab) I always get RuntimeError: CUDA error: device-side assert triggered, with huggingface. When I convert to CPU, it works fine, any solution?

Context: its solved, when I corrected the number of label classes, closing


LaBSE vs multilingual BERT, same layers?

Question:
Hey all am I missing something here:
Do the bert base multilingual uncased 1 and sentence-bert/LaBSE 3 have the same layers?
When I print out both models it seems so. I thought they are different? Is it just the data that they have been trained on that differs?
Thanks a lot

Context: Oh I see now, they indeed use the same layers as indicated in the research paper 2. (page 4)


Does transformers 3.5.1 support auto mixed precision training？

Question:
I’m using a GPTLMHead model in pytorch.
Is it possible , i add autocast() in the forward function in GPTLMHead and change the training process followed the Automatic Mixed Precision — PyTorch Tutorials 1.8.1+cu102 documentation 5

Context: Yes, it’s possible.


Regression is failing in fine tuning with BERT/GPT-2/Albert

Question:
I have been trying to use BertModel, albert and GPT2 models for fine tuning on my regression task and i was able to produce unwanted results . i will mention it below:
I tried it two times:

I used CLS token embeddings and fine tuned over my entire custom model but it produced some random number repeating over and over in my output matrix space.

I simply passed CLS token embeddings to the feed forward NN. In this case also it produced some random number.

what can be the solution to this problem?
class Custom_GPT(tf.keras.Model):

  def __init__(self,embedding_dim):

    super(Custom_GPT,self).__init__()

    self.embedding_dim=embedding_dim

    self.dense=tf.keras.layers.Dense(1,input_shape=(embedding_dim,),activation=None,name='dense_layer_1')

    self.GPT_layer=GPT_model

  def call(self,input_ids):

    sequence=self.GPT_layer(input_ids)[0]

    cls=sequence[:,0,:]

    x=self.dense(cls)

model doesn’t seem to be learning anything here. it generates a random constant repeatedly

Context: Are you returning x after call? I am not familiar with Tensorflow, but I assume that you still have to return the final logits? Otherwise it will implicitly return None.


Wav2Vec2 for Audio Emotion Classification

Question:
We are having a thesis project on Podcast Trailer Generation - Hotspot Detection for Podcast Dataset at Spotify.
The Spotify Podcast Dataset contains both transcript and audio data for many podcast episodes, and currently we are looking to use Wav2Vec2 embeddings as input to train an emotion classification model for the audio data. The audio data is currently only in English (with accompanied transcript).
It would be much appreciated if you could help out with fine-tuning Wav2Vec2 on some standard emotion-annotated audio datasets (e.g. RAVDESS 10, SAVEE 11). We will then use the fine-tuned embeddings as input for emotion classification, after which we will have human evaluation on the classified results.

Context: That sounds great. I’m also working with fine-tuning Wav2Vec2. I can help you out if you have any questions. @patrickvonplaten is also a great person to ask.


Question:
Hi HuggingFace Team
We are at the beginning of a new DL project. In this project, we work with transformers in TF2.
Before we start this we’d love to know what are your plans about creating a training framework for TF2.
In our search for answers, we came across a GitHub issue that mentioned that TFtrainer will be changed/removed.

      github.com/huggingface/transformers

      evaluation in TFTrainer does not run on GPU 1

        opened

        May 5, 2021

          rohanshingade

    ## Environment info
<!-- You can run the command `transformers-cli env` and cop…y-and-paste its output below.
     Don't forget to fill out the missing fields in that output! -->

- `transformers` version: 4.5.1
- Platform: ubuntu 18.04
- Python version: 3.6.9
- PyTorch version (GPU?):
- Tensorflow version (GPU?): tensorflow-gpu==2.4.1
- Using GPU in script?: Yes
- Using distributed or parallel set-up in script?: No

### Who can help
<!-- Your issue will be replied to more quickly if you can figure out the right person to tag with @
 If you know how to use git blame, that is the easiest way, otherwise, here is a rough guide of **who to tag**.
 Please tag fewer than 3 people.

Models:

- albert, bert, xlm: @LysandreJik
- blenderbot, bart, marian, pegasus, encoderdecoder,  t5: @patrickvonplaten, @patil-suraj
- longformer, reformer, transfoxl, xlnet: @patrickvonplaten
- fsmt: @stas00
- funnel: @sgugger
- gpt2: @patrickvonplaten, @LysandreJik
- rag: @patrickvonplaten, @lhoestq
- tensorflow: @Rocketknight1

Library:

- benchmarks: @patrickvonplaten
- deepspeed: @stas00
- ray/raytune: @richardliaw, @amogkam
- text generation: @patrickvonplaten
- tokenizers: @LysandreJik
- trainer: @sgugger
- pipelines: @LysandreJik

Documentation: @sgugger

Model hub:

- for issues with a model report at https://discuss.huggingface.co/ and tag the model's creator.

HF projects:

- datasets: [different repo](https://github.com/huggingface/datasets)
- rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)

Examples:

- maintained examples (not research project or legacy): @sgugger, @patil-suraj
- research_projects/bert-loses-patience: @JetRunner
- research_projects/distillation: @VictorSanh

 -->
@patil-suraj @Rocketknight1

## Information

I'm using TFT5ForConditionalGeneration for masked language modelling task. During training GPU utilisation is above 95% but as soon as evaluation starts it goes to 0%. Evaluation is slow. Even though [evaluate function is in strategy.scope()](https://github.com/huggingface/transformers/blob/c065025c4720a87783ca504a9018454893d00649/src/transformers/trainer_tf.py#L580). it does not use gpu.

The problem arises when using:
* [ ] the official example scripts: (give details below)

I'm using the official example script of TFTrainer and modified `run_tf_glue.py` a bit for custom data input.

The tasks I am working on is:
* [ ] my own task or dataset: (give details below)

Final train_dataset and eval_dataset (input to TFTrainer) have the form `({"input_ids": , "attention_mask": ,"decoder_attention_mask": }, labels)`

## To reproduce

Steps to reproduce the behavior:

I tried reproducing the error using run_tf_squad.py and run_tf_glue.py but both the scripts gave error as the inputs to the trainer were not compatible. Only MRPC task worked, but it had only 400 examples in evaluation so hard to determine. Rest of them simply didn't work, there was an error.

If possible I would like to contribute to TFTrainer in terms of running evaluation on GPU and processing squad and glue dataset to match dimensions to TFTrainer inputs. Guidance is really appreciated.

<!-- If you have code snippets, error messages, stack traces please provide them here as well.
     Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
     Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.-->

## Expected behavior

We were hoping you could shed more light on your plans for the integration of the transformers library with TF2. More concretely -
Do you intend to release a TF Trainer?
Will it be using Keras?
Any date expectations?
Thanks,

Context:
Hi there!
Yes the TFTrainer will be deprecated and removed in v5, we will focus on better integrating with Keras (though the means of Keras callbacks if we need to add functionality). Checkout the new classification example 7 for an example of where we are going.


Error when using the forward() function of `LongformerLayer` class

Question:
Hello,
Sorry if my question sounds a bit silly, but I just have a question:
I am trying to use LongformerForMultipleChoice model for a multiple-choice question that has 4 options.
When I do:
my_Longformer_multiple_choice_model.encoder.layer[layer_index].forward(hidden_output,
                                                 attention_mask=my_attention_mask,output_attention=False)

, an this error is generated:

  File "/Users/hyunjindominiquecho/opt/anaconda3/lib/python3.7/site-packages/transformers/modeling_longformer.py", line 384, in _sliding_chunks_query_key_matmul
    batch_size, seq_len, num_heads, head_dim = query.size()

ValueError: too many values to unpack (expected 4)

Here, my_attention_mask is the same attention mask that I would specify under the regular LongformerForMultipleChoice command:
# I am using the LongformerForMultipleChoice model, where each multiple choice question has 4 options.
my_attention_mask = tensor([[[1, 1, 1,  ..., 0, 0, 0],
         [1, 1, 1,  ..., 0, 0, 0],
         [1, 1, 1,  ..., 0, 0, 0],
         [1, 1, 1,  ..., 0, 0, 0]]])
# I can use the my_attention_mask in the regular command as below:
longformer_output= my_Longformer_multiple_choice_model(input_ids=input_ids,....,attention_mask=my_attention_mask)

why is this value error generated? What should I pass for the attention_mask parameter in the command my_Longformer_multiple_choice_model.encoder.layer[layer_index].forward(hidden_output,                                                   attention_mask,output_attention=False)?
Thank you,

Context: That seems to me error more related to the data structures rather than the longformer, may be you should check the format of input data first.


Question:
Hi,
I’m training roberta-base using HF Trainer, but it’s stuck at the starting itself. Here’s my code -
train_dataset[0]
{'input_ids': tensor([  0, 100, 657,  ...,   1,   1,   1]),
 'attention_mask': tensor([1, 1, 1,  ..., 0, 0, 0]),
 'labels': tensor(0)}

val_dataset[0]
{'input_ids': tensor([    0, 11094,    14,  ...,     1,     1,     1]),
 'attention_mask': tensor([1, 1, 1,  ..., 0, 0, 0]),
 'labels': tensor(0)}

## simple test
model(train_dataset[:2]['input_ids'], attention_mask = train_dataset[:2]['attention_mask'], labels=train_dataset[:2]['labels'])
SequenceClassifierOutput(loss=tensor(0.6995, grad_fn=<NllLossBackward>), logits=tensor([[ 0.0438, -0.1893],
        [ 0.0530, -0.1786]], grad_fn=<AddmmBackward>), hidden_states=None, attentions=None)

train_args = transformers.TrainingArguments(
             output_dir='test_1',
             overwrite_output_dir=True,
             evaluation_strategy="epoch",
             per_device_train_batch_size=8,
             per_device_eval_batch_size=8,
             learning_rate=3e-5,
             weight_decay=0.01,
             num_train_epochs=2,
             load_best_model_at_end=True,
             )

trainer = transformers.Trainer(
             model=model,
             args=train_args,
             train_dataset=train_dataset,
             eval_dataset=val_dataset,
             tokenizer=tok,
             )

trainer.train()

I saw memory consumption and it is stuck at -
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.51.06    Driver Version: 450.51.06    CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla V100-SXM2...  On   | 00000000:62:00.0 Off |                    0 |
| N/A   49C    P0    60W / 300W |   1756MiB / 32510MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
|   1  Tesla V100-SXM2...  On   | 00000000:8A:00.0 Off |                    0 |
| N/A   50C    P0    61W / 300W |   1376MiB / 32510MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+

Plz let me know how to proceed further…

Context: any progress since then?


Question:
Suppose I have a sequence that consists of 2 sentences separated by </SEP> tokens like A </SEP> B. When performing forward pass with RoBERTa model, I want tokens in sentence A only attend to tokens in sentence A and vice versa for sentence B. The mask will be look like this:

In summary, is there any way to explicitly pass a custom attention mask to the model?
Thanks in advance.

Context:
Attention mask is normally created from input_mask . You cannot bypass attention mask directly. I might be wrong also.
For your purpose, create an input_mask with 1s on First row rows and two colums and then 1s on last two rows and last two columns. Set else to 0.


Issues with building extensions in Deepspeed

Question:
I get this error even after following the instructions on Deepspeed installation page and HF Trainer docs. Can anyone suggest how to fix this ?
I’ve followed @stas’s replies on GH issues, but this error keeps coming up. pybind11 is installed on my machine, not sure what is this indicating ?

ds1305×675 166 KB

Context:
@prajjwal1 - something is off in your python env.
The mantra should always be - let’s see if others have asked this question already, so here try https://www.google.com/search?q=pybind11%2Fpybind11.h+no+such+file+or+directory 12
Please avoid posting images for tracebacks - instead copy-n-paste the output using code blocks - this is because it’s impossible to copy-n-paste from the image to do the search for you.
That’s said if the issue continues after you tried the solutions offered at the top matching pages - please post the details at Issues · microsoft/DeepSpeed · GitHub


How to train your own corpus without labels

Question:
Hi, I’d like to fine-tune BERT for my product catalog corpus which contains a lot of out-of-vocabulary words like brand names. By fine-tuning, I mean transfer learning and not training from scratch.
I have been following this [Fine-tuning a pretrained model — transformers 4.5.0.dev0 documentation 11] tutorial and see that this requires labels. As you can imagine, my use case if connected to information retrieval and search and does not contain any y_labels. All I want is unique vector embedding out of my trained model.
How should I approach this problem using HF Trainer module?

Context:
hey @awaiskaleem if i understand correctly, i that what you’re looking for is to fine-tune the language model on your corpus. this will generally produce mask-filling that more accurately captures the relations in your corpus and for BERT you can check out the Masked language modeling section of this tutorial: Google Colaboratory 79
once the language model is fine-tune, you can save the weights and then load them using AutoModel.from_pretrained to generate your embeddings.


Ask for help with prediction results of Named Entity Recognition Task

Question:
Hi guys,
After training the NER Task with using RoBERTa Architecture, I got the below result

{‘eval_loss’: 0.003242955543100834,
‘eval_precision’: 0.9959672534053343,
‘eval_recall’: 0.9959672534053343,
‘eval_f1’: 0.9959672534053343,
‘eval_accuracy’: 0.9995624335836689}

The result generally is quite high, as I expected. But here is my confusion, when I randomly input a set of sentences (out of the training set) to really know the model’s performance.
My pseudo code
    def tokenize_and_align_labels_random(examples, tokenizer):
        tokenized_inputs = tokenizer(examples['tokens'],
                                     truncation=True,
                                     is_split_into_words=True)
        return tokenized_inputs
    def preprocess_datasets(tokenizer, **datasets) -> Dict[str, Dataset]:
        tokenize_ner = partial(tokenize_and_align_labels_random,
                               tokenizer=tokenizer)
        return {k: ds.map(tokenize_ner) for k, ds in datasets.items()}

     address=Testing_Dataset[Testing_Dataset['address']==1]['text'].apply(clean_doc).tolist()

    da_datasets_random_Test = preprocess_datasets(tokenizer,
    test=Dataset.from_dict({'tokens':address}))

    results=da_trainer.predict(da_datasets_random_Test['test'])

    predictions=results.predictions
    predictions = np.argmax(predictions, axis=2)
    # Remove ignored index (special tokens)
    true_predictions = [
        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]

I input the sentences with some words that don’t exist in the tokenizer vocabulary, and the model will handle that part for me by automatically generating their sub token.

That means the ‘input_ids’ will generate more token ids for presenting these cases, the problem is their predicted tags will also be increasing (based on how many tokens was delivered to the model).

For instance

Input sentence: “Giao tôi lê_lai phường hai tân_bình hcm”
Value after tokenizer:
{‘input_ids’: [0, 64003, 64003, 17489, 6115, 64139, 64151, 64003, 6446, 64313, 1340, 74780, 2], ‘token_type_ids’: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ‘attention_mask’: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
Because tokenize of “lê_lai” is [‘lê@@’, ‘l@@’, ‘ai’]; of “tân_bình” is ['tân@@’, ‘bình’]; of “hcm” is [‘h@@’, ‘cm’]

The result I got after all: [‘O’,‘O’,‘B-LOC’,‘I-LOC’,‘I-LOC’,‘I-LOC’,‘I-LOC’,‘I-LOC’,‘O’,‘I-LOC’,‘I-LOC’, ‘O’]
In fact, their prediction should only have 7 tags for the input tokens, but now it was more than this. So do guys have any strategies for this (I got one that we can train the tokenizer with more tokens).
I do appreciate your time and sharing.

Context:
Hello Iacle.
I suspect your issue is around WordPiece Tokenization but I can’t tell for sure with the info you posted.
Take a look at
Fine-tuning with custom datasets — transformers 4.5.0.dev0 documentation 9
In particular pay attention to when it talks about WordPiece Tokenization and the code in
def encode_tags(tags, encodings): …


What’s a good value for pad_to_multiple_of?

Question:
Have anyone tried this? I can’t find a suggested value in doc.
I am running on GPU (with tensor core).

Context:
If you use mixed precision, you need all your tensors to have dimensions that are multiple of 8s to maximize the benefits of your tensor cores.
So pas_to_multiple_of=8 is a good value, unless you model has some pooling (like Funnel Transformer) in which case those 8 might be divided by 2 (you’d need pad_to_multiple_of=32 for this model for instance, since there are two pooling operations).


Architecture attribute of model.config is different from the actual model’s architecture in RoBERTa

Question:
Loading roberta-base model using RobertaForSequenceClassification.from_pretrained returns a model having config with incorrect value for attribute architectures (“architectures”: [    “RobertaForMaskedLM”  ]) of RobertaForMaskedLM instead of RobertaForSequenceClassification.
Similarly, I tried for RobertaForMultipleChoice and TFRobertaForSequenceClassification and got same result of in appropriate architecture attribute.
(below are code snippets and their outputs)
Can anyone explain the reason for this?

package versions:
transformers-4.6.0
tensorflow-2.4.1
torch-1.8.1+cu101

from transformers import RobertaTokenizer, RobertaForSequenceClassification
import torch
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
model = RobertaForSequenceClassification.from_pretrained('roberta-base')
model.config

Output:
Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.bias', 'roberta.pooler.dense.weight', 'lm_head.bias', 'lm_head.dense.bias']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
RobertaConfig {
  "_name_or_path": "roberta-base",
  "architectures": [
    "RobertaForMaskedLM"
  ],
  "attention_probs_dropout_prob": 0.1,
  "bos_token_id": 0,
  "eos_token_id": 2,
  "gradient_checkpointing": false,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 768,
  "initializer_range": 0.02,
  "intermediate_size": 3072,
  "layer_norm_eps": 1e-05,
  "max_position_embeddings": 514,
  "model_type": "roberta",
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "pad_token_id": 1,
  "position_embedding_type": "absolute",
  "transformers_version": "4.6.0",
  "type_vocab_size": 1,
  "use_cache": true,
  "vocab_size": 50265
}

from transformers import RobertaTokenizer, RobertaForMultipleChoice
import torch
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
model = RobertaForMultipleChoice.from_pretrained('roberta-base')
model.config

Output:

Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForMultipleChoice: ['lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.dense.bias']
- This IS expected if you are initializing RobertaForMultipleChoice from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForMultipleChoice from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForMultipleChoice were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
RobertaConfig {
  "_name_or_path": "roberta-base",
  "architectures": [
    "RobertaForMaskedLM"
  ],
  "attention_probs_dropout_prob": 0.1,
  "bos_token_id": 0,
  "eos_token_id": 2,
  "gradient_checkpointing": false,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 768,
  "initializer_range": 0.02,
  "intermediate_size": 3072,
  "layer_norm_eps": 1e-05,
  "max_position_embeddings": 514,
  "model_type": "roberta",
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "pad_token_id": 1,
  "position_embedding_type": "absolute",
  "transformers_version": "4.6.0",
  "type_vocab_size": 1,
  "use_cache": true,
  "vocab_size": 50265
}

from transformers import RobertaTokenizer, TFRobertaForSequenceClassification
import tensorflow as tf
model = TFRobertaForSequenceClassification.from_pretrained('roberta-base')
model.config

Output:
All model checkpoint layers were used when initializing TFRobertaForSequenceClassification.

Some layers of TFRobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
RobertaConfig {
  "_name_or_path": "roberta-base",
  "architectures": [
    "RobertaForMaskedLM"
  ],
  "attention_probs_dropout_prob": 0.1,
  "bos_token_id": 0,
  "eos_token_id": 2,
  "gradient_checkpointing": false,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 768,
  "initializer_range": 0.02,
  "intermediate_size": 3072,
  "layer_norm_eps": 1e-05,
  "max_position_embeddings": 514,
  "model_type": "roberta",
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "pad_token_id": 1,
  "position_embedding_type": "absolute",
  "transformers_version": "4.6.0",
  "type_vocab_size": 1,
  "use_cache": true,
  "vocab_size": 50265
}

Context: This is the architecture of the model class it was pretrained with. If you save your model (preferably after training) using the save_pretrained method, this architectures field will be updated to put the architecture you were using.


Question:
Hello everyone,
I try to use pegasus on squad dataset. I am using notebook 2 for pegasus fine-tuning on xsum.
I use all hyper parameters as in the above notebook. After train, I got following loss values.

/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:562: UserWarning: This overload of add_ is deprecated: add_(Number alpha, Tensor other) Consider using one of the following signatures instead: add_(Tensor other, *, Number alpha) (Triggered internally at /pytorch/torch/csrc/utils/python_arg_parser.cpp:1005.) exp_avg_sq_row.mul_(beta2t).add_(1.0 - beta2t, update.mean(dim=-1))
[2000/2000 46:21, Epoch 2000/2000]

Step
Training Loss
Validation Loss

100
11.515400
11.971568

200
9.583800
10.400596

300
7.782900
8.812467

400
1.653800
1.603077

500
0.006000
0.367762

600
0.049500
0.271221

700
0.000600
0.271198

800
0.000300
0.282740

900
0.000200
0.311012

1000
0.000200
0.308879

1100
0.000100
0.316374

1200
0.000100
0.329436

1300
0.000100
0.326996

1400
0.020200
0.291157

1500
0.002400
0.272543

1600
0.000100
0.291934

1700
0.035900
0.308235

1800
0.000100
0.308755

1900
0.000100
0.311096

2000
0.000000
0.311382

CPU times: user 1h 1min 52s, sys: 18min 49s, total: 1h 20min 41s Wall time: 46min 26s
TrainOutput(global_step=2000, training_loss=1.8480298345236805, metrics={‘train_runtime’: 2783.2468, ‘train_samples_per_second’: 0.719, ‘total_flos’: 0, ‘epoch’: 2000.0, ‘init_mem_cpu_alloc_delta’: 8192, ‘init_mem_gpu_alloc_delta’: 0, ‘init_mem_cpu_peaked_delta’: 0, ‘init_mem_gpu_peaked_delta’: 0, ‘train_mem_cpu_alloc_delta’: -4224876544, ‘train_mem_gpu_alloc_delta’: 2288025088, ‘train_mem_cpu_peaked_delta’: 4242300928, ‘train_mem_gpu_peaked_delta’: 8677123584})

My question is what values for hyperparameters (optimization, learning rate etc.) I should use for a QA task? And the training loss shown in TrainOutput does not match with the loss values on table. What can the reason be?

Context:
hey @helloworld123-lab, are you sure that pegasus can be used for a reading comprehension task like SQuAD? my understanding is that it is designed for abstractive tasks like summarization (although i’d be interested to hear otherwise!).
as an alternative, i’d suggest checking out the official question-answering tutorial here: Google Colaboratory 3


Help Improving Abstractive Summarization

Question:
Hey everyone,
Hope you’re doing great!
I’m a beginner when it comes to using transformers.
I’ve been working on book summarization project for a while, the idea is to split the book into chapters then the chapter into chunks and summarize the chunks separately.
I’ve tried several models and the summaries provided aren’t that good. Some of the problems are:

Some sentences aren’t fully generated.
The context is lost most of the time.

Do you guys have any suggestions please?
Also, I have found these two papers:

Mind The Facts: Knowledge-Boosted Coherent Abstractive Text Summarization 4
 Enhancing Factual Consistency of Abstractive Summarization 4

But I couldn’t find any code implementation of these models. If you have any implementations please let me know.
Thank you for your time!
Have a good day.

Context:
hey @haithembrr, your approach sounds very sensible
have you tried playing around with the parameters of the model’s generate function, e.g. max_len and trying different strategies like beam search vs sampling (see docs 1)?
alternatively, you could have a look at the discussion in this thread to see if someone has also run into the same problem: Summarization on long documents 13


Checkpoint missing Optimizer.pt? How to Resume?

Question:
I tried to train a model with HF and it helped me a lot! My only problem is resuming the training. As you can see in the screenshot below, only my first checkpoint contains the data I expect. My question is, is there a flag where I can turn off saving the checkpoints (I ask only to turn it off!)? Can I still continue the training?

image306×566 19 KB

Im using load_best_model_at_end save_total_limit = 3 overwrite_output_dir
I didnt change my Code i just updated to using the latest HF Version
!pip install -q git+https://github.com/huggingface/transformers
Is there any way to resum from the last Checkpoint? Maybe a Flag init_epoch etc?
TrainingArguments(output_dir=/share/datasets/output_run, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.STEPS, prediction_loss_only=False, per_device_train_batch_size=20, per_device_eval_batch_size=16, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=0.0001, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=20.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/May12_05-06-46_a600ce861ff7, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=1000, save_strategy=IntervalStrategy.STEPS, save_steps=1000, save_total_limit=3, no_cuda=False, seed=42, fp16=True, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=1000, dataloader_num_workers=2, past_index=-1, run_name=cv_sm_1, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=True, metric_for_best_model=loss, greater_is_better=False, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=True, length_column_name=length, report_to=['wandb'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, _n_gpu=1, mp_parameters=)

!find / -name optimizer.pt

Just returned the Checkpoint from withing the Screenshot

Context:
@sgugger
do you by any chance know what this could be?
The Strange thing is, i resumed training from last valid checkpoint and the model normally generates checkpoints as expected, but at some point the checkpoint folder only contains rng_state.pth as shown on the screenshot.


Masked vectors are included in vanilla transformer model

Question:
I trained a vanilla Bert Model.
model = BertModel.from_pretrained(‘bert-base-uncased’, num_labels = 4,
output_hidden_states=True,
output_attentions=True)
That I trained. I’m using multi label classification, so I train this model with an extra logit layer attached to it, then summed the results. Code for running it is below
def validation_nn():
    prob_list = []
    input_ids = [[101,4769, 77, 102, 0, 0]]
    mask = [[1, 1, 1, 1, 0, 0]]

    model.eval()

    outputs = model(torch.tensor(input_ids), attention_mask=torch.tensor(mask) )

    last_hidden_state = outputs.last_hidden_state

    summed_final_hidden_state = torch.sum(last_hidden_state, 1)
    logits = logit_layer(summed_final_hidden_state)

    probs = torch.sigmoid(logits)

    prob_list.append(probs)
    print(outputs[-1])
    #print('probs', probs)

    return prob_list

probs = validation_nn()

Here is a snippet of the output

tensor([[[[0.2019, 0.1652, 0.0762, 0.5566, 0.0000, 0.0000],
[0.2206, 0.1417, 0.3824, 0.2552, 0.0000, 0.0000],
[0.0319, 0.1476, 0.6395, 0.1810, 0.0000, 0.0000],
[0.4228, 0.1063, 0.1553, 0.3157, 0.0000, 0.0000],
[0.1468, 0.1533, 0.4655, 0.2344, 0.0000, 0.0000],
[0.1618, 0.1286, 0.4877, 0.2219, 0.0000, 0.0000]],
     [[0.9799, 0.0092, 0.0029, 0.0080, 0.0000, 0.0000],
      [0.0043, 0.0714, 0.9032, 0.0212, 0.0000, 0.0000],
      [0.2281, 0.1369, 0.4741, 0.1608, 0.0000, 0.0000],
      [0.3400, 0.3117, 0.0927, 0.2557, 0.0000, 0.0000],
      [0.0053, 0.1026, 0.7928, 0.0992, 0.0000, 0.0000],
      [0.0063, 0.0944, 0.8022, 0.0971, 0.0000, 0.0000]],

As you can see, the two columns to the far right, which should be masked, are zero’d out, as they should be. But I get two more rows which are not zero’d out. When I go to sum my results, these mask vectors are added, which obviously messes up my results. I imagine having these non zero’d vectors messes up the training as well. Is there a way to zero out the rows, which belong to mask tokens?

Context:
hey @bennicholl you might have better luck using the Trainer class to run your training with a custom loss function (see docs):
import torch
from transformers import Trainer

class MultilabelTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        labels = inputs.pop("labels")
        outputs = model(**inputs)
        logits = outputs.logits
        loss_fct = torch.nn.BCEWithLogitsLoss()
        loss = loss_fct(logits.view(-1, self.model.config.num_labels),
                        labels.float().view(-1, self.model.config.num_labels))
        return (loss, outputs) if return_outputs else loss

then you can define a compute_metrics function as follows
from scipy.special import expit as sigmoid
from sklearn.metrics import classification_report

def compute_metrics(pred):
    y_true = pred.label_ids
    y_pred = sigmoid(pred.predictions) y_pred = (y_pred>0.5).astype(float)

    clf_dict = classification_report(y_true, y_pred, target_names=all_labels,
                                         zero_division=0, output_dict=True)
    return {"micro f1": clf_dict['micro avg']['f1-score'], "macro f1": clf_dict['macro avg']['f1-score']}

and after fine-tuning you can then get the predictions on your validation set via Trainer.predict:
trainer = # fine-tuned Trainer
pred = trainer.predict(your_eval_dataset)
metrics = compute_metrics(pred)

hth!


Question:
Hello,
I am trying to do summarization with mT5 and when I use the official summarization colab which uses seq2seq trainer, the model outputs trash. You can see my GitHub issue here 56. Do you have any ideas on how to proceed with summarization using mT5?

Context: disabling fp16 seems to solve the issue, but I would rather have fp16 because it doubles the training speed


ELECTRA: Accounting for mask tokens that are correctly predicted by MLM

Question:
I have been pretraining ELECTRA models using the ElectraForPreTraining class, some of them taking in input_ids and others just taking the embeddings straight via the input_embeds parameter (I am training the model on biological data and assign particular tokens to particular species).
As I understand from the paper, during pretraining both the generator and discriminator are trained; the generator is a small masked language model and the discriminator aims to predict whether a token is an original or a token replaced by the generator. Additionally, as the paper states " if the generator happens to generate the correct token, that token is considered “real” instead of “fake”. However, the labels ElectraForPreTraining uses to determine the ground truth of “real” or “fake” are an array passed in by me - and I have no way of knowing whether the generator will correctly predict the token or not, resulting in the labels I pass in marking any masked tokens as “fake” whether or not the generator actually succeeds. Additionally, since the model doesn’t take in any information regarding the true identity of masked tokens it seems to me that the model has no way to handle this case. Is this being properly handled in the model? If so, what do I have to do to make sure I am passing in my data correctly?

Context:
inputs
| mask
masked inputs
| generator (Electraformlm)
gen logits
| gumbel softmax
generated
| discriminator(electraformpretraining)
disc logits
is_replaced = generated == inputs(x)
edited: is_replaced = generated != inputs
is replaced is the true label for discriminator


Question:
Hello,
When the following code is run several times (notebook language_modeling.ipynb 3), it gives a diferent value at each time:
import math
eval_results = trainer.evaluate()
print(f"Perplexity: {math.exp(eval_results['eval_loss']):.2f}")

I do not understand why (the eval loss should be always the same when using the same eval dataset and the same model).

Context: I dont think its the exact same model and the same evaluation dataset everytime. Can you actually save and check the eval dataset across two runs please


How to set early stopping when running run_summarization.py

Question: Same with the topic. I know how to change the training epochs buut I don’t know how to add early stopping when running run_summarization.py

Context:
Hi, I’m having a similar issue. I’m trying to set an EarlyStoppingCallback in the run_mlm.py parameters setting the following flag:
--callbacks [EarlyStoppingCallback(early_stopping_patience=3)]
When doing this bash complains about the '('. I have tried to use a more bash-friendly syntax adding \ before the parentheses but it says that that it is not used in the HfArgumentParser.
Have you figured out how to set it?
Thank you very much!!


Wav2vec2-large-xlsr-53 for non-listed low resource language

Question:
Hi,
Can we fine-tuned the model for low resource language not inside the 53 languages listed?
Does it have to be inside the same language family of the 53 listed?
Regards
Becks

Context:
Hello,
Yes you can fine-tune the model for low resource language outside the ones which the model was pretrained on. On the original paper of XLSR-53 ([2006.13979] Unsupervised Cross-lingual Representation Learning for Speech Recognition 1) you can see on table 3 and table 4 that they finetuned on out-of-pretraining languages.
In addition, during my school project, we finetuned XLSR-53 ont the Czech and Ukrainian language and got some good results , (feel free to  check my github -  GitHub - omarsou/wav2vec_xlsr_cv_exp: Experiments on out of training languages (from common voice https://commonvoice.mozilla.org/) using Wav2Vec 11 ).
Best,
Omar


Question:
I was wondering how one might go about generating text on a machine with multiple GPU? Here is some example code:
import pandas as pd
from transformers import GPT2Tokenizer, GPT2LMHeadModel

tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

def generate_text(primer_sent: str, chunk_sent_length: int, is_finetuning_current_model=None):
    encoded_results = tokenizer(primer_sent, return_tensors='pt')
    tmp_ids = encoded_results['input_ids']
    max_length = tmp_ids.shape[1] + chunk_sent_length
    gen_ids = model.generate(input_ids=tmp_ids,
                                  max_length=max_length,
                                  is_finetuning_current_model=is_finetuning_current_model,
                                  do_sample=True,
                                  top_k=50,
                                  top_p=0.95,
                                  pad_token_id=50256)
    decoded_str = tokenizer.decode(gen_ids, skip_special_tokens=True)
    return decoded_str

def generate_text_samples(data_filename: str, chunk_sent_length: int):
    sent_df = pd.read_csv(data_filename)
    sent_df['gen'] = sent_df[0].apply(generate_text, args=(chunk_sent_length,))
    return sent_df

text_df = generate_text_samples('/path/to/data.csv', 50)

When I run this code on my machine, it appears that it is all being done on the CPU. Is it possible to distribute this workflow across the multiple gpu I have? Thanks in advance for your help!

Context: Hi @aclifton314. Have you had any luck with this? I’m also interested in generating text (using a different model - t5) on multiple GPUs to speed up the throughput of my process. Wondering if you were able to do this?


Conda install -c huggingface or conda-forge?

Question:
Hello,
In the Transformers docs, the conda installation paragraph 10 gives the following code that installs the version 4.4.2:
conda install -c huggingface transformers
… but the Anaconda page of Transformers 6 gives the following one that installs the version 4.5.1 (the latest one):
conda install -c conda-forge transformers
Why the Hugging Face channel does not install the latest version?

Context:
Hi @pierreguillou, there were some build errors with the conda builds, they’ve since been fixed and the packages have been uploaded.
You should now see v4.5.0 and v4.5.1 on the huggingface channel.