from transformers import T5TokenizerFast
from datasets import load_dataset
import torch
​
#tokenizer = T5TokenizerFast.from_pretrained("t5-small")
​
def preprocess_multi_task(examples):
    if examples["answer"] is None or examples["context"] is None:
        return None
​
    if not isinstance(examples["answer"], list) or not isinstance(examples["context"], list):
        return None
​
    if len(examples["answer"]) != len(examples["context"]):
        return None
​
    prepared_input_texts = []
    prepared_questions = []
    prepared_contexts = []
    prepared_decoder_input_ids = []  # New list to hold decoder input ids
    
    for context, answer in zip(examples["context"], examples["answer"]):
        input_format = f"generate question: context: {context} answer: {answer}"
        prepared_input_texts.append(input_format)
        prepared_questions.append(f"What is the question for this context: '{context}' and answer: '{answer}'?")
        prepared_contexts.append(context)
        
        # Generate decoder input ids
        decoder_input_ids = tokenizer.encode(f"generate question: context: {context} answer: {answer}", return_tensors="pt")
        
        # Pad decoder input ids to match the maximum length of the input_ids
        decoder_input_ids = torch.nn.functional.pad(decoder_input_ids, (0, 128 - decoder_input_ids.shape[1]), value=tokenizer.pad_token_id)
        
        prepared_decoder_input_ids.append(decoder_input_ids.squeeze())  # Remove the batch dimension
​
    tokenized_inputs = tokenizer(
        prepared_input_texts,
        max_length=128,
        padding="max_length",
        truncation=True,
    )
    
    tokenized_questions = tokenizer(
        prepared_questions,
        max_length=64,
        padding="max_length",
        truncation=True,
    )
    
    # Add decoder_input_ids to tokenized inputs
    tokenized_inputs["decoder_input_ids"] = prepared_decoder_input_ids
​
    return tokenized_inputs
​
# Apply preprocessing to datasets
tokenized_train_dataset = train_dataset.map(preprocess_multi_task, batched=True)
tokenized_val_dataset = val_dataset.map(preprocess_multi_task, batched=True)
tokenized_test_dataset = test_dataset.map(preprocess_multi_task, batched=True)
​
100%
43/43 [00:55<00:00, 1.24s/ba]
100%
11/11 [00:14<00:00, 1.21s/ba]
100%
6/6 [00:07<00:00, 1.27s/ba]
add Codeadd Markdown
from transformers import T5ForConditionalGeneration, Trainer, TrainingArguments, DataCollatorForSeq2Seq
​
model = T5ForConditionalGeneration.from_pretrained("UBC-NLP/AraT5v2-base-1024")
​
# Define data collator for sequence-to-sequence models
data_collator = DataCollatorForSeq2Seq(tokenizer)
​
# Define the training arguments
training_args = TrainingArguments(
    output_dir="./question_generation_arat1024_2_model",
    num_train_epochs=2,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    save_total_limit=2,
    evaluation_strategy="epoch",
    report_to=[],  # Set report_to to an empty list to disable wandb logging
    learning_rate=5e-5  # Set your desired learning rate here
)
​
# Model trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_train_dataset,
    eval_dataset=tokenized_val_dataset,
    tokenizer=tokenizer,
    data_collator=data_collator,
)
​
# Train the model
trainer.train()
​
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[46], line 31
     21 trainer = Trainer(
     22     model=model,
     23     args=training_args,
   (...)
     27     data_collator=data_collator,
     28 )
     30 # Train the model
---> 31 trainer.train()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1624, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1622         hf_hub_utils.enable_progress_bars()
   1623 else:
-> 1624     return inner_training_loop(
   1625         args=args,
   1626         resume_from_checkpoint=resume_from_checkpoint,
   1627         trial=trial,
   1628         ignore_keys_for_eval=ignore_keys_for_eval,
   1629     )

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1961, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1958     self.control = self.callback_handler.on_step_begin(args, self.state, self.control)
   1960 with self.accelerator.accumulate(model):
-> 1961     tr_loss_step = self.training_step(model, inputs)
   1963 if (
   1964     args.logging_nan_inf_filter
   1965     and not is_torch_tpu_available()
   1966     and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))
   1967 ):
   1968     # if loss is nan or inf simply add the average of previous logged losses
   1969     tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2902, in Trainer.training_step(self, model, inputs)
   2899     return loss_mb.reduce_mean().detach().to(self.args.device)
   2901 with self.compute_loss_context_manager():
-> 2902     loss = self.compute_loss(model, inputs)
   2904 if self.args.n_gpu > 1:
   2905     loss = loss.mean()  # mean() to average on multi-gpu parallel training

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2943, in Trainer.compute_loss(self, model, inputs, return_outputs)
   2941 else:
   2942     if isinstance(outputs, dict) and "loss" not in outputs:
-> 2943         raise ValueError(
   2944             "The model did not return a loss from the inputs, only the following keys: "
   2945             f"{','.join(outputs.keys())}. For reference, the inputs it received are {','.join(inputs.keys())}."
   2946         )
   2947     # We don't use .loss here since the model may return tuples instead of ModelOutput.
   2948     loss = outputs["loss"] if isinstance(outputs, dict) else outputs[0]

ValueError: The model did not return a loss from the inputs, only the following keys: logits,past_key_values,encoder_last_hidden_state. For reference, the inputs it received are input_ids,attention_mask,decoder_input_ids.