Repository: UKPLab/sentence-transformers
Issue category: Bug Report

Issue:
`eval_dataset` or `evaluator`?
The docs say

```
You can use both an eval_dataset and an evaluator, one or the other, or neither. They evaluate based on the eval_strategy and eval_steps [Training Arguments](https://www.sbert.net/docs/sentence_transformer/training_overview.html#training-arguments).
```

But when I omit `eval_dataset` and only pass an `evaluator` into my args I get this from `SentenceTransformerTrainer`:

```
File ~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:419, in Trainer.__init__(self, model, args, data_collator, train_dataset, eval_dataset, processing_class, model_init, compute_loss_func, compute_metrics, callbacks, optimizers, preprocess_logits_for_metrics)
    [413](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:413)         raise ValueError(
    [414](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:414)             "When using `batch_eval_metrics`, your `compute_metrics` function must take a `compute_result`"
    [415](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:415)             " boolean argument which will be triggered after the last batch of the eval set to signal that the"
    [416](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:416)             " summary statistics should be returned by the function."
    [417](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:417)         )
    [418](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:418) if args.eval_strategy is not None and args.eval_strategy != "no" and eval_dataset is None:
--> [419](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:419)     raise ValueError(
    [420](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:420)         f"You have set `args.eval_strategy` to {args.eval_strategy} but you didn't pass an `eval_dataset` to `Trainer`. Either set `args.eval_strategy` to `no` or pass an `eval_dataset`. "
    [421](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:421)     )
    [422](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:422) self.args = args
    [423](https://vscode-remote+wsl-002bubuntu.vscode-resource.vscode-cdn.net/home/deklan/code/investor_company_str_pairs/notebooks/~/code/investor_company_str_pairs/.venv/lib/python3.12/site-packages/transformers/trainer.py:423) self.compute_loss_func = compute_loss_func

ValueError: You have set `args.eval_strategy` to IntervalStrategy.STEPS but you didn't pass an `eval_dataset` to `Trainer`. Either set `args.eval_strategy` to `no` or pass an `eval_dataset`.
```


My versions are

```
    "sentence-transformers>=3.2.1",
    "transformers[torch]>=4.46.1",
```


Hints:
Downgrading to `transformers==4.44.0` seems to have fixed
Hello!

Thanks for reporting - this is indeed likely an issue with the newest `transformers` version(s). I'll try and chase it down and include a fix in the next release. I'll keep you in the loop.

- Tom Aarsen
The original PR that introduced this problem: https://github.com/huggingface/transformers/pull/33743

Patch:
diff --git a/sentence_transformers/trainer.py b/sentence_transformers/trainer.py
index 3fd20eb1f..2a7907e3a 100644
--- a/sentence_transformers/trainer.py
+++ b/sentence_transformers/trainer.py
@@ -209,7 +209,7 @@ def __init__(
             "args": args,
             "data_collator": data_collator,
             "train_dataset": train_dataset,
-            "eval_dataset": eval_dataset,
+            "eval_dataset": eval_dataset if eval_dataset is not None or evaluator is None else "dummy",
             "model_init": model_init,
             "compute_metrics": compute_metrics,
             "callbacks": callbacks,
@@ -222,6 +222,12 @@ def __init__(
         else:
             super_kwargs["tokenizer"] = tokenizer
         super().__init__(**super_kwargs)
+        # Transformers v4.46.0 introduced a ValueError if `eval_dataset` is None while eval_strategy is not "no",
+        # but in Sentence Transformers you can also evaluate without an eval_dataset via an evaluator, so we set
+        # it to "dummy" in that case to avoid the ValueError
+        if self.eval_dataset == "dummy":
+            self.eval_dataset = None
+
         # Every Sentence Transformer model can always return a loss, so we set this to True
         # to avoid having to specify it in the data collator or model's forward
         self.can_return_loss = True


Test patch:
diff --git a/tests/test_trainer.py b/tests/test_trainer.py
index 64fbf827e..e7e9827d5 100644
--- a/tests/test_trainer.py
+++ b/tests/test_trainer.py
@@ -8,8 +8,10 @@
 
 import pytest
 import torch
+from datasets.dataset_dict import DatasetDict
 
 from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
+from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator
 from sentence_transformers.training_args import SentenceTransformerTrainingArguments
 from sentence_transformers.util import is_datasets_available, is_training_available
 from tests.utils import SafeTemporaryDirectory
@@ -230,3 +232,47 @@ def test_trainer(
         original_embeddings = original_model.encode("The cat is on the mat.", convert_to_tensor=True)
         new_embeddings = model.encode("The cat is on the the mat.", convert_to_tensor=True)
         assert not torch.equal(original_embeddings, new_embeddings)
+
+
+@pytest.mark.parametrize("use_eval_dataset", [True, False])
+@pytest.mark.parametrize("use_evaluator", [True, False])
+def test_trainer_no_eval_dataset_with_eval_strategy(
+    stsb_bert_tiny_model: SentenceTransformer,
+    stsb_dataset_dict: DatasetDict,
+    use_eval_dataset: bool,
+    use_evaluator: bool,
+    tmp_path: Path,
+) -> None:
+    # Expect a crash when `args.eval_strategy` is not "no" but neither `eval_dataset` or `evaluator` is provided
+    # Otherwise, the trainer should be created without any issues
+    model = stsb_bert_tiny_model
+    train_dataset = stsb_dataset_dict["train"].select(range(10))
+    eval_dataset = stsb_dataset_dict["validation"].select(range(10))
+    evaluator = EmbeddingSimilarityEvaluator(
+        sentences1=eval_dataset["sentence1"],
+        sentences2=eval_dataset["sentence2"],
+        scores=[score / 5 for score in eval_dataset["score"]],
+        name="stsb-validation",
+    )
+    loss = losses.CosineSimilarityLoss(model=model)
+    args = SentenceTransformerTrainingArguments(output_dir=tmp_path, eval_strategy="steps")
+
+    kwargs = {}
+    if use_eval_dataset:
+        kwargs["eval_dataset"] = eval_dataset
+    if use_evaluator:
+        kwargs["evaluator"] = evaluator
+
+    if not use_eval_dataset and not use_evaluator:
+        context = pytest.raises(ValueError, match=".*`args.eval_strategy`.*")
+    else:
+        context = nullcontext()
+
+    with context:
+        SentenceTransformerTrainer(
+            model=model,
+            args=args,
+            train_dataset=train_dataset,
+            loss=loss,
+            **kwargs,
+        )