​
# Tokenization
tokenizer = BertTokenizerFast.from_pretrained('aubmindlab/bert-base-arabert')
# Initialize Bert2Bert from pre-trained checkpoints
model = EncoderDecoderModel.from_encoder_decoder_pretrained('aubmindlab/bert-base-arabert', 'aubmindlab/bert-base-arabert')
​
#tokenizer = AutoTokenizer.from_pretrained("UBC-NLP/AraT5-base")#flax-community/arabic-t5-small")
#model = AutoModelForSeq2SeqLM.from_pretrained("UBC-NLP/AraT5-base")#flax-community/arabic-t5-small")
tokenizer_config.json: 100%
637/637 [00:00<00:00, 52.5kB/s]
vocab.txt: 100%
717k/717k [00:00<00:00, 44.9MB/s]
tokenizer.json: 100%
2.26M/2.26M [00:00<00:00, 5.97MB/s]
special_tokens_map.json: 100%
112/112 [00:00<00:00, 7.20kB/s]
config.json: 100%
578/578 [00:00<00:00, 39.3kB/s]
model.safetensors: 100%
543M/543M [00:01<00:00, 394MB/s]
Some weights of BertLMHeadModel were not initialized from the model checkpoint at aubmindlab/bert-base-arabert and are newly initialized: ['bert.encoder.layer.8.crossattention.self.key.weight', 'bert.encoder.layer.5.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.self.value.weight', 'bert.encoder.layer.10.crossattention.self.query.weight', 'bert.encoder.layer.1.crossattention.output.dense.weight', 'bert.encoder.layer.0.crossattention.self.value.bias', 'bert.encoder.layer.1.crossattention.self.key.weight', 'bert.encoder.layer.3.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.0.crossattention.self.key.weight', 'bert.encoder.layer.2.crossattention.output.dense.weight', 'bert.encoder.layer.9.crossattention.self.value.weight', 'bert.encoder.layer.4.crossattention.self.query.bias', 'bert.encoder.layer.0.crossattention.self.query.weight', 'bert.encoder.layer.4.crossattention.output.dense.weight', 'bert.encoder.layer.4.crossattention.self.query.weight', 'bert.encoder.layer.8.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.2.crossattention.output.dense.bias', 'bert.encoder.layer.11.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.self.value.weight', 'bert.encoder.layer.9.crossattention.output.dense.weight', 'bert.encoder.layer.5.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.self.key.bias', 'bert.encoder.layer.4.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.1.crossattention.self.query.bias', 'bert.encoder.layer.7.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.8.crossattention.self.key.bias', 'bert.encoder.layer.4.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.2.crossattention.self.value.weight', 'bert.encoder.layer.6.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.self.value.weight', 'bert.encoder.layer.6.crossattention.self.query.bias', 'bert.encoder.layer.5.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.self.query.bias', 'bert.encoder.layer.4.crossattention.self.key.weight', 'bert.encoder.layer.4.crossattention.self.key.bias', 'bert.encoder.layer.6.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.3.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.output.dense.weight', 'bert.encoder.layer.2.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.10.crossattention.self.value.weight', 'bert.encoder.layer.0.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.self.value.weight', 'bert.encoder.layer.8.crossattention.output.dense.bias', 'bert.encoder.layer.8.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.10.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.output.dense.bias', 'bert.encoder.layer.3.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.self.key.weight', 'bert.encoder.layer.9.crossattention.self.query.weight', 'bert.encoder.layer.8.crossattention.self.query.bias', 'bert.encoder.layer.7.crossattention.output.dense.weight', 'bert.encoder.layer.2.crossattention.self.query.weight', 'bert.encoder.layer.2.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.4.crossattention.output.dense.bias', 'bert.encoder.layer.3.crossattention.self.key.weight', 'bert.encoder.layer.1.crossattention.output.dense.bias', 'bert.encoder.layer.10.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.7.crossattention.output.dense.bias', 'bert.encoder.layer.0.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.1.crossattention.self.value.weight', 'bert.encoder.layer.10.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.self.query.weight', 'bert.encoder.layer.6.crossattention.self.value.bias', 'bert.encoder.layer.0.crossattention.self.key.bias', 'bert.encoder.layer.11.crossattention.self.query.weight', 'bert.encoder.layer.5.crossattention.output.dense.weight', 'bert.encoder.layer.1.crossattention.self.key.bias', 'bert.encoder.layer.9.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.self.key.weight', 'bert.encoder.layer.5.crossattention.self.key.weight', 'bert.encoder.layer.8.crossattention.self.query.weight', 'bert.encoder.layer.3.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.5.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.output.dense.weight', 'bert.encoder.layer.1.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.output.dense.bias', 'bert.encoder.layer.3.crossattention.self.value.weight', 'bert.encoder.layer.5.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.3.crossattention.self.query.bias', 'bert.encoder.layer.2.crossattention.self.query.bias', 'bert.encoder.layer.3.crossattention.self.key.bias', 'bert.encoder.layer.2.crossattention.self.value.bias', 'bert.encoder.layer.9.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.7.crossattention.self.query.bias', 'bert.encoder.layer.3.crossattention.self.query.weight', 'bert.encoder.layer.5.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.self.key.bias', 'bert.encoder.layer.2.crossattention.self.key.bias', 'bert.encoder.layer.5.crossattention.self.query.bias', 'bert.encoder.layer.9.crossattention.self.value.bias', 'bert.encoder.layer.10.crossattention.self.key.bias', 'bert.encoder.layer.2.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.self.key.weight', 'bert.encoder.layer.4.crossattention.self.value.bias', 'bert.encoder.layer.7.crossattention.self.value.bias', 'bert.encoder.layer.3.crossattention.output.dense.weight', 'bert.encoder.layer.1.crossattention.self.value.bias', 'bert.encoder.layer.5.crossattention.self.value.bias', 'bert.encoder.layer.1.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.6.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.0.crossattention.self.value.weight', 'bert.encoder.layer.11.crossattention.self.query.bias', 'bert.encoder.layer.7.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.9.crossattention.self.query.bias', 'bert.encoder.layer.10.crossattention.self.key.weight', 'bert.encoder.layer.0.crossattention.output.dense.weight', 'bert.encoder.layer.9.crossattention.output.dense.bias', 'bert.encoder.layer.6.crossattention.self.key.bias', 'bert.encoder.layer.4.crossattention.self.value.weight', 'bert.encoder.layer.0.crossattention.self.query.bias', 'bert.encoder.layer.1.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.10.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.0.crossattention.output.dense.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
add Codeadd Markdown
# Load model directly
#from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
​
#tokenizer = AutoTokenizer.from_pretrained("MIIB-NLP/Arabic-question-generation")
#model = AutoModelForSeq2SeqLM.from_pretrained("MIIB-NLP/Arabic-question-generation")
add Codeadd Markdown
# Define a preprocess function for the question generation task
def preprocess_function(examples):
    inputs = tokenizer(
        examples["positive"],
        max_length=128,
        truncation=True,
        padding="max_length",
        return_offsets_mapping=True,
    )
    targets = tokenizer(
        examples["query"],
        max_length=64,  # Adjust as needed
        truncation=True,
        padding="max_length",
        return_offsets_mapping=True,
    )
​
    inputs["labels"] = targets["input_ids"]
    inputs["decoder_attention_mask"] = targets["attention_mask"]
    return inputs
add Codeadd Markdown
# Apply preprocess function to train, val, and test datasets
tokenized_train_dataset = train_dataset.map(preprocess_function, batched=True)
tokenized_val_dataset = val_dataset.map(preprocess_function, batched=True)
tokenized_test_dataset = test_dataset.map(preprocess_function, batched=True)
100%
185/185 [11:40<00:00, 2.76s/ba]
100%
21/21 [01:17<00:00, 3.18s/ba]
100%
52/52 [03:13<00:00, 2.81s/ba]
add Codeadd Markdown
# Define data collator for sequence-to-sequence models
data_collator = DataCollatorForSeq2Seq(tokenizer)#, model_type="t5")
add Codeadd Markdown
from transformers import EarlyStoppingCallback
​
# Model training arguments
training_args = TrainingArguments(
    output_dir="./question_generation_model",
    num_train_epochs=1,
    auto_find_batch_size=True,
    logging_steps=15000,
    save_steps=15000,
    #warmup_steps=500,
    #weight_decay=0.01,
    #logging_dir="./logs",
    #logging_steps=10,
    learning_rate=1e-5,
    evaluation_strategy="steps",  # Change evaluation strategy to "steps"
    report_to=[],  # Set report_to to an empty list to disable wandb logging
    load_best_model_at_end=True
)
​
# Create the EarlyStoppingCallback
early_stopping = EarlyStoppingCallback(early_stopping_patience=3, early_stopping_threshold=0.005)
​
# Model trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    data_collator=lambda data: {
        "input_ids": torch.stack([item["input_ids"] for item in data]),
        "attention_mask": torch.stack([item["attention_mask"] for item in data]),
        "labels": torch.stack([item["labels"] for item in data]),
    },
    callbacks=[early_stopping],  # Add the EarlyStoppingCallback
)
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[13], line 28
     21 early_stopping = EarlyStoppingCallback(early_stopping_patience=3, early_stopping_threshold=0.005)
     23 # Model trainer
     24 trainer = Trainer(
     25     model=model,
     26     args=training_args,
     27     train_dataset=train_dataset,
---> 28     eval_dataset=eval_dataset,
     29     data_collator=lambda data: {
     30         "input_ids": torch.stack([item["input_ids"] for item in data]),
     31         "attention_mask": torch.stack([item["attention_mask"] for item in data]),
     32         "labels": torch.stack([item["labels"] for item in data]),
     33     },
     34     callbacks=[early_stopping],  # Add the EarlyStoppingCallback
     35 )

NameError: name 'eval_dataset' is not defined
add Codeadd Markdown
from transformers import EarlyStoppingCallback
​
# Model training arguments
training_args = TrainingArguments(
    output_dir="./question_generation_bert",
    num_train_epochs=3,
    auto_find_batch_size=True,
    logging_steps=15000,
    save_steps=15000,
    #warmup_steps=500,
    #weight_decay=0.01,
    #logging_dir="./logs",
    #logging_steps=10,
    learning_rate=1e-5,
    evaluation_strategy="steps",  # Change evaluation strategy to "steps"
    report_to=[],  # Set report_to to an empty list to disable wandb logging
    load_best_model_at_end=True
)
​
# Create the EarlyStoppingCallback
early_stopping = EarlyStoppingCallback(early_stopping_patience=3, early_stopping_threshold=0.005)
​
# Model trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_train_dataset,
    eval_dataset=tokenized_val_dataset,
    tokenizer=tokenizer,
    data_collator=data_collator,
    #optimizer=optimizer,
    #compute_metrics=compute_metrics,
    #callbacks=[early_stopping],  # Add the EarlyStoppingCallback
)
add Codeadd Markdown
trainer.train()
You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[16], line 1
----> 1 trainer.train()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1537, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1535         hf_hub_utils.enable_progress_bars()
   1536 else:
-> 1537     return inner_training_loop(
   1538         args=args,
   1539         resume_from_checkpoint=resume_from_checkpoint,
   1540         trial=trial,
   1541         ignore_keys_for_eval=ignore_keys_for_eval,
   1542     )

File /opt/conda/lib/python3.10/site-packages/accelerate/utils/memory.py:136, in find_executable_batch_size.<locals>.decorator(*args, **kwargs)
    134     raise RuntimeError("No executable batch size found, reached zero.")
    135 try:
--> 136     return function(batch_size, *args, **kwargs)
    137 except Exception as e:
    138     if should_reduce_batch_size(e):

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1854, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1851     self.control = self.callback_handler.on_step_begin(args, self.state, self.control)
   1853 with self.accelerator.accumulate(model):
-> 1854     tr_loss_step = self.training_step(model, inputs)
   1856 if (
   1857     args.logging_nan_inf_filter
   1858     and not is_torch_tpu_available()
   1859     and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))
   1860 ):
   1861     # if loss is nan or inf simply add the average of previous logged losses
   1862     tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2735, in Trainer.training_step(self, model, inputs)
   2732     return loss_mb.reduce_mean().detach().to(self.args.device)
   2734 with self.compute_loss_context_manager():
-> 2735     loss = self.compute_loss(model, inputs)
   2737 if self.args.n_gpu > 1:
   2738     loss = loss.mean()  # mean() to average on multi-gpu parallel training

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2758, in Trainer.compute_loss(self, model, inputs, return_outputs)
   2756 else:
   2757     labels = None
-> 2758 outputs = model(**inputs)
   2759 # Save past state if it exists
   2760 # TODO: this needs to be fixed and made cleaner later.
   2761 if self.args.past_index >= 0:

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/transformers/models/encoder_decoder/modeling_encoder_decoder.py:616, in EncoderDecoderModel.forward(self, input_ids, attention_mask, decoder_input_ids, decoder_attention_mask, encoder_outputs, past_key_values, inputs_embeds, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, **kwargs)
    613     encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states)
    615 if (labels is not None) and (decoder_input_ids is None and decoder_inputs_embeds is None):
--> 616     decoder_input_ids = shift_tokens_right(
    617         labels, self.config.pad_token_id, self.config.decoder_start_token_id
    618     )
    619     if decoder_attention_mask is None:
    620         decoder_attention_mask = decoder_input_ids.new_tensor(decoder_input_ids != self.config.pad_token_id)

File /opt/conda/lib/python3.10/site-packages/transformers/models/encoder_decoder/modeling_encoder_decoder.py:158, in shift_tokens_right(input_ids, pad_token_id, decoder_start_token_id)
    156 shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
    157 if decoder_start_token_id is None:
--> 158     raise ValueError("Make sure to set the decoder_start_token_id attribute of the model's configuration.")
    159 shifted_input_ids[:, 0] = decoder_start_token_id
    161 if pad_token_id is None:

ValueError: Make sure to set the decoder_start_token_id attribute of the model's configuration.  how fix error