Update notebook.py to support multi eval datasets
Author: matrix1001
Pull request: yes

# Fix key error when using multiple evaluation datasets

## Code triggering the error
Any code using multiple eval_dataset will trigger the error.
```py
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset={
        'valid':valid_dataset,
        'test':test_dataset,
    },
    compute_metrics=compute_metrics
)
```
## Before fix
![image](https://github.com/huggingface/transformers/assets/32097922/86c83a11-49a6-4315-8c1f-3affde96fe6a)

Here's detailed error msg:
```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[1], line 80
     68 model = BertForRelationExtraction.from_pretrained('bert-base-cased', config=config)
     69 trainer = Trainer(
     70     model=model,
     71     args=training_args,
   (...)
     77     compute_metrics=compute_metrics
     78 )
---> 80 trainer.train()

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/trainer.py:1664], in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1659     self.model_wrapped = self.model
   1661 inner_training_loop = find_executable_batch_size(
   1662     self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size
   1663 )
-> 1664 return inner_training_loop(
   1665     args=args,
   1666     resume_from_checkpoint=resume_from_checkpoint,
   1667     trial=trial,
   1668     ignore_keys_for_eval=ignore_keys_for_eval,
   1669 )

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/trainer.py:2019], in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   2016     self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch
   2017     self.control = self.callback_handler.on_step_end(args, self.state, self.control)
-> 2019     self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
   2020 else:
   2021     self.control = self.callback_handler.on_substep_end(args, self.state, self.control)

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/trainer.py:2293], in Trainer._maybe_log_save_evaluate(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)
   2291     metrics = {}
   2292     for eval_dataset_name, eval_dataset in self.eval_dataset.items():
-> 2293         dataset_metrics = self.evaluate(
   2294             eval_dataset=eval_dataset,
   2295             ignore_keys=ignore_keys_for_eval,
   2296             metric_key_prefix=f"eval_{eval_dataset_name}",
   2297         )
   2298         metrics.update(dataset_metrics)
   2299 else:

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/trainer.py:3057], in Trainer.evaluate(self, eval_dataset, ignore_keys, metric_key_prefix)
   3053 if DebugOption.TPU_METRICS_DEBUG in self.args.debug:
   3054     # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
   3055     xm.master_print(met.metrics_report())
-> 3057 self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, output.metrics)
   3059 self._memory_tracker.stop_and_update_metrics(output.metrics)
   3061 return output.metrics

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/trainer_callback.py:379], in CallbackHandler.on_evaluate(self, args, state, control, metrics)
    377 def on_evaluate(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, metrics):
    378     control.should_evaluate = False
--> 379     return self.call_event("on_evaluate", args, state, control, metrics=metrics)

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/trainer_callback.py:397], in CallbackHandler.call_event(self, event, args, state, control, **kwargs)
    395 def call_event(self, event, args, state, control, **kwargs):
    396     for callback in self.callbacks:
--> 397         result = getattr(callback, event)(
    398             args,
    399             state,
    400             control,
    401             model=self.model,
    402             tokenizer=self.tokenizer,
    403             optimizer=self.optimizer,
    404             lr_scheduler=self.lr_scheduler,
    405             train_dataloader=self.train_dataloader,
    406             eval_dataloader=self.eval_dataloader,
    407             **kwargs,
    408         )
    409         # A Callback can skip the return of `control` if it doesn't change it.
    410         if result is not None:

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/utils/notebook.py:350], in NotebookProgressCallback.on_evaluate(self, args, state, control, metrics, **kwargs)
    348         name = " ".join([part.capitalize() for part in splits[1:]])
    349         values[name] = v
--> 350 self.training_tracker.write_line(values)
    351 self.training_tracker.remove_child()
    352 self.prediction_bar = None

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/utils/notebook.py:244], in NotebookTrainingTracker.write_line(self, values)
    242             columns.append(key)
    243     self.inner_table[0] = columns
--> 244 self.inner_table.append([values[c] for c in columns])

File [~/anaconda3/envs/pytorch2/lib/python3.11/site-packages/transformers/utils/notebook.py:244](, in (.0)
    242             columns.append(key)
    243     self.inner_table[0] = columns
--> 244 self.inner_table.append([values[c] for c in columns])

KeyError: 'Valid Accuracy'
```
## After fix
![image](https://github.com/huggingface/transformers/assets/32097922/1a1b9e4c-67a1-49d7-8935-11dd3ad35974)


## Some explanation
I remove all predefined key "Validation Loss". However, this will have the following result if there is only one eval_dataset:
![image](https://github.com/huggingface/transformers/assets/32097922/2e770530-833f-46c1-b17a-c0eaac8eea8e)
I think we don't have to name it as "Validation Loss"?

My modification allows dynamic columns and updates values if multiple calls for `NotebookProgressCallback.on_evaluate` correspond to the same epoch or step.



Comments:
Note that my last commit corresponds to the workflow check. I dont think it's necessary.

The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/transformers/pr_25796). All of your documentation changes will be reflected on that endpoint.

@matrix1001 do you want me to merge this or do you still have modifications to include?

@ArthurZucker merge.

```
import evaluate

f1_metric = evaluate.load("f1")
precision_metric = evaluate.load("precision")
recall_metric = evaluate.load("recall")
accuracy_metric = evaluate.load("accuracy")
average_method = 'weighted'
def compute_metrics(eval_pred):
    results = {}
    predictions = np.argmax(eval_pred.predictions, axis=1)
    labels = eval_pred.label_ids
    results.update(f1_metric.compute(predictions=predictions, references = labels, average=average_method))
    results.update(precision_metric.compute(predictions=predictions, references = labels, average=average_method))
    results.update(recall_metric.compute(predictions=predictions, references = labels, average=average_method))
    results.update(accuracy_metric.compute(predictions=predictions, references = labels))
    return results
```

In this code, on master branch I am still seeing:

```
----> 1 train_results = trainer.train()

File ~/.local/lib/python3.8/site-packages/transformers/trainer.py:1591, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1589         hf_hub_utils.enable_progress_bars()
   1590 else:
-> 1591     return inner_training_loop(
   1592         args=args,
   1593         resume_from_checkpoint=resume_from_checkpoint,
   1594         trial=trial,
   1595         ignore_keys_for_eval=ignore_keys_for_eval,
   1596     )

File ~/.local/lib/python3.8/site-packages/transformers/trainer.py:1999, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1996     self.control.should_training_stop = True
   1998 self.control = self.callback_handler.on_epoch_end(args, self.state, self.control)
-> 1999 self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
   2001 if DebugOption.TPU_METRICS_DEBUG in self.args.debug:
   2002     if is_torch_tpu_available():
   2003         # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)

File ~/.local/lib/python3.8/site-packages/transformers/trainer.py:2339, in Trainer._maybe_log_save_evaluate(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)
   2336         self.lr_scheduler.step(metrics[metric_to_check])
   2338 if self.control.should_save:
-> 2339     self._save_checkpoint(model, trial, metrics=metrics)
   2340     self.control = self.callback_handler.on_save(self.args, self.state, self.control)

File ~/.local/lib/python3.8/site-packages/transformers/trainer.py:2458, in Trainer._save_checkpoint(self, model, trial, metrics)
   2456 if not metric_to_check.startswith("eval_"):
   2457     metric_to_check = f"eval_{metric_to_check}"
-> 2458 metric_value = metrics[metric_to_check]
   2460 operator = np.greater if self.args.greater_is_better else np.less
   2461 if (
   2462     self.state.best_metric is None
   2463     or self.state.best_model_checkpoint is None
   2464     or operator(metric_value, self.state.best_metric)
   2465 ):

KeyError: 'eval_accuracy'
```

@puneetdabulya This PR only fixes `notebook.py`. You may need to fix it in another PR.