Repository: pykeen/pykeen
Issue: IndexError when training model with TextRepresentation
Root cause: Validation Gap

### Describe the bug

Trying to train model with TextRepresentation and while other models work as usual there seems to be a strange issue with tokeniser.

```
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[17], line 1
----> 1 result = pipeline(
      2     model=ERModel,
      3     training=training,
      4     testing=testing,
      5     validation=testing,
      6     stopper='early',
      7     model_kwargs=dict(
     8         interaction="ermlpe",
     9         interaction_kwargs=dict(
     10             embedding_dim=entity_representations.shape[0],
     11         ),
     12         entity_representations=entity_representations,
     13         relation_representations_kwargs=dict(
     14             shape=entity_representations.shape,
     15         ),
     16     ),
     17     training_kwargs=dict(
     18         batch_size=64,
     19         num_epochs=1,
     20     ),
     21     evaluation_kwargs=dict(
     22         batch_size=11
     23     ),
     24     stopper_kwargs=dict(frequency=3, patience=3, relative_delta=0.0001),
     25     random_seed=42,
     26     negative_sampler_kwargs=dict(
     27         filtered=True,
     28     filterer='python-set',  
     29         num_negs_per_pos = 3, 
     30         corruption_scheme=(0)
     31     ),
     32 )

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/pipeline/api.py:1368, in pipeline(dataset, dataset_kwargs, training, testing, validation, evaluation_entity_whitelist, evaluation_relation_whitelist, model, model_kwargs, interaction, interaction_kwargs, dimensions, loss, loss_kwargs, regularizer, regularizer_kwargs, optimizer, optimizer_kwargs, clear_optimizer, lr_scheduler, lr_scheduler_kwargs, training_loop, training_loop_kwargs, negative_sampler, negative_sampler_kwargs, epochs, training_kwargs, stopper, stopper_kwargs, evaluator, evaluator_kwargs, evaluation_kwargs, result_tracker, result_tracker_kwargs, metadata, device, random_seed, use_testing_data, evaluation_fallback, filter_validation_when_testing, use_tqdm)
   1359 _result_tracker.log_params(
   1360     params=dict(
   1361         evaluation_kwargs={
   (...)
   1365     )
   1366 )
   1367 evaluate_start_time = time.time()
-> 1368 metric_results: MetricResults = _safe_evaluate(
   1369     model=model_instance,
   1370     mapped_triples=mapped_triples,
   1371     evaluator=evaluator_instance,
   1372     evaluation_kwargs=evaluation_kwargs,
   1373     evaluation_fallback=evaluation_fallback,
   1374 )
   1375 evaluate_end_time = time.time() - evaluate_start_time
   1376 _result_tracker.log_metrics(metrics=dict(final_evaluation=evaluate_end_time), step=step, prefix="times")

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/pipeline/api.py:1424, in _safe_evaluate(model, mapped_triples, evaluator, evaluation_kwargs, evaluation_fallback)
   1422 while True:
   1423     try:
-> 1424         metric_results: MetricResults = evaluator.evaluate(
   1425             model=model,
   1426             mapped_triples=mapped_triples,
   1427             **evaluation_kwargs,
   1428         )
   1429     except (MemoryError, RuntimeError) as e:
   1430         # If the evaluation still fail using the CPU, the error is raised
   1431         if model.device.type != "cuda" or not evaluation_fallback:

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:208, in Evaluator.evaluate(self, model, mapped_triples, batch_size, slice_size, **kwargs)
    205         # Clear the ranks from the current evaluator
    206         self.finalize()
--> 208 rv = evaluate(
    209     model=model,
    210     mapped_triples=mapped_triples,
    211     evaluator=self,
    212     batch_size=batch_size,
    213     slice_size=slice_size,
    214     **kwargs,
    215 )
    216 # Since squeeze is true, we can expect that evaluate returns a MetricResult, but we need to tell MyPy that
    217 return cast(MetricResults, rv)

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:682, in evaluate(model, mapped_triples, evaluator, only_size_probing, batch_size, slice_size, device, use_tqdm, tqdm_kwargs, restrict_entities_to, restrict_relations_to, do_time_consuming_checks, additional_filter_triples, pre_filtered_triples, targets, mode)
    680 relation_filter = None
    681 for target in targets:
--> 682     relation_filter = _evaluate_batch(
    683         batch=batch,
    684         model=model,
    685         target=target,
    686         evaluator=evaluator,
    687         slice_size=slice_size,
    688         all_pos_triples=all_pos_triples,
    689         relation_filter=relation_filter,
    690         restrict_entities_to=restrict_entities_to,
    691         mode=mode,
    692     )
    694 # If we only probe sizes we do not need more than one batch
    695 if only_size_probing and evaluated_once:

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:755, in _evaluate_batch(batch, model, target, evaluator, slice_size, all_pos_triples, relation_filter, restrict_entities_to, mode)
    715 def _evaluate_batch(
    716     batch: MappedTriples,
    717     model: Model,
   (...)
    725     mode: Optional[InductiveMode],
    726 ) -> torch.BoolTensor:
    727     """
    728     Evaluate ranking for batch.
    729 
   (...)
    753         The relation filter, which can be re-used for the same batch.
    754     """
--> 755     scores = model.predict(hrt_batch=batch, target=target, slice_size=slice_size, mode=mode)
    757     if evaluator.filtered or evaluator.requires_positive_mask:
    758         column = TARGET_TO_INDEX[target]

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/models/base.py:465, in Model.predict(self, hrt_batch, target, full_batch, ids, **kwargs)
    463     if full_batch:
    464         hrt_batch = hrt_batch[:, 0:2]
--> 465     return self.predict_t(hrt_batch, **kwargs, tails=ids)
    467 if target == LABEL_RELATION:
    468     if full_batch:

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/models/base.py:403, in Model.predict_t(self, hr_batch, **kwargs)
    401 self.eval()  # Enforce evaluation mode
    402 hr_batch = self._prepare_batch(batch=hr_batch, index_relation=1)
--> 403 scores = self.score_t(hr_batch, **kwargs)
    404 if self.predict_with_sigmoid:
    405     scores = torch.sigmoid(scores)

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/models/nbase.py:466, in ERModel.score_t(self, hr_batch, slice_size, mode, tails)
    464 # add broadcast dimension
    465 hr_batch = hr_batch.unsqueeze(dim=1)
--> 466 h, r, t = self._get_representations(h=hr_batch[..., 0], r=hr_batch[..., 1], t=tails, mode=mode)
    467 # unsqueeze if necessary
    468 if tails is None or tails.ndimension() == 1:

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/models/nbase.py:570, in ERModel._get_representations(self, h, r, t, mode)
    568 head_representations = [head_representations[i] for i in self.interaction.head_indices()]
    569 tail_representations = [tail_representations[i] for i in self.interaction.tail_indices()]
--> 570 hr, rr, tr = [
    571     [representation(indices=indices) for representation in representations]
    572     for indices, representations in (
    573         (h, head_representations),
    574         (r, self.relation_representations),
    575         (t, tail_representations),
    576     )
    577 ]
    578 # normalization
    579 return cast(
    580     Tuple[HeadRepresentation, RelationRepresentation, TailRepresentation],
    581     tuple(x[0] if len(x) == 1 else x for x in (hr, rr, tr)),
    582 )

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/models/nbase.py:571, in <listcomp>(.0)
    568 head_representations = [head_representations[i] for i in self.interaction.head_indices()]
    569 tail_representations = [tail_representations[i] for i in self.interaction.tail_indices()]
    570 hr, rr, tr = [
--> 571     [representation(indices=indices) for representation in representations]
    572     for indices, representations in (
    573         (h, head_representations),
    574         (r, self.relation_representations),
    575         (t, tail_representations),
    576     )
    577 ]
    578 # normalization
    579 return cast(
    580     Tuple[HeadRepresentation, RelationRepresentation, TailRepresentation],
    581     tuple(x[0] if len(x) == 1 else x for x in (hr, rr, tr)),
    582 )

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/models/nbase.py:571, in <listcomp>(.0)
    568 head_representations = [head_representations[i] for i in self.interaction.head_indices()]
    569 tail_representations = [tail_representations[i] for i in self.interaction.tail_indices()]
    570 hr, rr, tr = [
--> 571     [representation(indices=indices) for representation in representations]
    572     for indices, representations in (
    573         (h, head_representations),
    574         (r, self.relation_representations),
    575         (t, tail_representations),
    576     )
    577 ]
    578 # normalization
    579 return cast(
    580     Tuple[HeadRepresentation, RelationRepresentation, TailRepresentation],
    581     tuple(x[0] if len(x) == 1 else x for x in (hr, rr, tr)),
    582 )

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/nn/representation.py:179, in Representation.forward(self, indices)
    177 if indices is not None and self.unique:
    178     indices, inverse = indices.unique(return_inverse=True)
--> 179 x = self._plain_forward(indices=indices)
    180 # normalize *before* repeating
    181 if self.normalizer is not None:

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/nn/representation.py:1046, in TextRepresentation._plain_forward(self, indices)
   1044 else:
   1045     labels = [self.labels[i] for i in indices.tolist()]
-> 1046 return self.encoder(labels=labels)

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/nn/text.py:77, in TextEncoder.forward(self, labels)
     75 labels = upgrade_to_sequence(labels)
     76 labels = list(map(str, labels))
---> 77 return self.forward_normalized(texts=labels)

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/pykeen/nn/text.py:223, in TransformerTextEncoder.forward_normalized(self, texts)
    221 def forward_normalized(self, texts: Sequence[str]) -> torch.FloatTensor:  # noqa: D102
    222     return self.model(
--> 223         **self.tokenizer(
    224             texts,
    225             return_tensors="pt",
    226             padding=True,
    227             truncation=True,
    228             max_length=self.max_length,
    229         ).to(get_preferred_device(self.model))
    230     ).pooler_output

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2520, in PreTrainedTokenizerBase.__call__(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)
   2518     if not self._in_target_context_manager:
   2519         self._switch_to_input_mode()
-> 2520     encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)
   2521 if text_target is not None:
   2522     self._switch_to_target_mode()

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2606, in PreTrainedTokenizerBase._call_one(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)
   2601         raise ValueError(
   2602             f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
   2603             f" {len(text_pair)}."
   2604         )
   2605     batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
-> 2606     return self.batch_encode_plus(
   2607         batch_text_or_text_pairs=batch_text_or_text_pairs,
   2608         add_special_tokens=add_special_tokens,
   2609         padding=padding,
   2610         truncation=truncation,
   2611         max_length=max_length,
   2612         stride=stride,
   2613         is_split_into_words=is_split_into_words,
   2614         pad_to_multiple_of=pad_to_multiple_of,
   2615         return_tensors=return_tensors,
   2616         return_token_type_ids=return_token_type_ids,
   2617         return_attention_mask=return_attention_mask,
   2618         return_overflowing_tokens=return_overflowing_tokens,
   2619         return_special_tokens_mask=return_special_tokens_mask,
   2620         return_offsets_mapping=return_offsets_mapping,
   2621         return_length=return_length,
   2622         verbose=verbose,
   2623         **kwargs,
   2624     )
   2625 else:
   2626     return self.encode_plus(
   2627         text=text,
   2628         text_pair=text_pair,
   (...)
   2644         **kwargs,
   2645     )

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2797, in PreTrainedTokenizerBase.batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)
   2787 # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
   2788 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
   2789     padding=padding,
   2790     truncation=truncation,
   (...)
   2794     **kwargs,
   2795 )
-> 2797 return self._batch_encode_plus(
   2798     batch_text_or_text_pairs=batch_text_or_text_pairs,
   2799     add_special_tokens=add_special_tokens,
   2800     padding_strategy=padding_strategy,
   2801     truncation_strategy=truncation_strategy,
   2802     max_length=max_length,
   2803     stride=stride,
   2804     is_split_into_words=is_split_into_words,
   2805     pad_to_multiple_of=pad_to_multiple_of,
   2806     return_tensors=return_tensors,
   2807     return_token_type_ids=return_token_type_ids,
   2808     return_attention_mask=return_attention_mask,
   2809     return_overflowing_tokens=return_overflowing_tokens,
   2810     return_special_tokens_mask=return_special_tokens_mask,
   2811     return_offsets_mapping=return_offsets_mapping,
   2812     return_length=return_length,
   2813     verbose=verbose,
   2814     **kwargs,
   2815 )

File /home/user/conda/envs/python_38/lib/python3.8/site-packages/transformers/tokenization_utils_fast.py:462, in PreTrainedTokenizerFast._batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose)
    455 # Convert the output to have dict[list] from list[dict] and remove the additional overflows dimension
    456 # From (variable) shape (batch, overflows, sequence length) to ~ (batch * overflows, sequence length)
    457 # (we say ~ because the number of overflow varies with the example in the batch)
    458 #
    459 # To match each overflowing sample with the original sample in the batch
    460 # we add an overflow_to_sample_mapping array (see below)
    461 sanitized_tokens = {}
--> 462 for key in tokens_and_encodings[0][0].keys():
    463     stack = [e for item, _ in tokens_and_encodings for e in item[key]]
    464     sanitized_tokens[key] = stack

IndexError: list index out of range
```

### How to reproduce

```python
from pykeen.nn.representation import TextRepresentation
from pykeen.models import ERModel


entity_representations = TextRepresentation.from_triples_factory(
    triples_factory=training, 
    encoder="transformer",
    encoder_kwargs=dict(pretrained_model_name_or_path="bert-base-multilingual-uncased", max_length=512),
)

result = pipeline(
    model=ERModel,
    training=training,
    testing=testing,
    validation=testing,
    stopper='early',
    model_kwargs=dict(
        interaction="ermlpe",
        interaction_kwargs=dict(
            embedding_dim=entity_representations.shape[0],
        ),
        entity_representations=entity_representations,
        relation_representations_kwargs=dict(
            shape=entity_representations.shape,
        ),
    ),
    training_kwargs=dict(
        batch_size=64,
        num_epochs=1,
    ),
    evaluation_kwargs=dict(
        batch_size=11
    ),
    random_seed=42,
    negative_sampler_kwargs=dict(
        filtered=True,
        filterer='python-set',  
        num_negs_per_pos = 3, 
        corruption_scheme=(0)
    ),
)
```

### Environment

| Key             | Value                    |
|-----------------|--------------------------|
| OS              | posix                    |
| Platform        | Linux                    |
| Release         | 5.4.0-122-generic        |
| Time            | Thu Jan 12 10:11:36 2023 |
| Python          | 3.8.15                   |
| PyKEEN          | 1.9.0                    |
| PyKEEN Hash     | UNHASHED                 |
| PyKEEN Branch   |                          |
| PyTorch         | 1.13.1+cu117             |
| CUDA Available? | true                     |
| CUDA Version    | 11.7                     |
| cuDNN Version   | 8500                     |

### Additional information

_No response_

### Issue Template Checks

- [X] This is not a feature request (use a different issue template if it is)
- [X] This is not a question (use the [discussions](https://github.com/pykeen/pykeen/discussions/new) forum instead)
- [X] I've read the text explaining why including environment information is important and understand if I omit this information that my issue will be dismissed

From reading the trace back, this looks like an error upstream in the encoder related to not having an appropriate way to encode a character in the string. It’s hard to tell without seeing the dataset used. Can you try and isolate the triple that’s causing this?

Okay, I'll try to find this triple, however, the error persists no matter what model I use and only during the evaluation

It’s no surprise that this error would persist for all models - the entity representation is where the problem is happening, not the interaction function. If you know its an issue with evaluation then at least that tells you where to look first 

The triplet appeared to be empty, although this thing didn't happen for models with default representations

I printed all the tensors that go through encoder and it seems that the some tensors are encodings or something.

```python
    # docstr-coverage: inherited
    def _plain_forward(
        self,
        indices: Optional[torch.LongTensor] = None,
    ) -> torch.FloatTensor:  # noqa: D102
        if indices is None:
            labels = self.labels
        else:
            print("****")
            print(indices)
            labels = [self.labels[i] for i in indices.tolist()]
        return self.encoder(labels=labels)
```

The tensors


****
```python
tensor([3129, 3162, 3173, 3228, 3365, 3485, 3924, 3928, 3941, 3952, 3988],
       device='cuda:0')
```
****
```python
tensor([], device='cuda:0', dtype=torch.int64)
[]
```




This is strange in two ways:

1. I'm not sure how there can be empty tensor of indices passed
2. The encoder should be a bit smarter about passing through empty lists

Yeah, I completely agree. Just in case, I checked whether there are any empty strings and there are none of them in triplets

Th transformers version is 4.21.3. I also tried the latest one but it didn't help

This seems to be the model & tokenizer you are using from huggingface's transformer library: https://huggingface.co/bert-base-multilingual-uncased.  For debugging, you could try to instantiate the tokenizer separately, and apply it to all entity labels to if any of them results in an empty tensor

```python
# replace this first part by loading your triples
from pykeen.datasets import Nations
training = Nations().training

# find entity labels which result in empty token sequences
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-multilingual-uncased")
for label in training.entity_to_id.keys():
    encoded_input = tokenizer(label, return_tensors="pt")
    input_ids = encoded_input["input_ids"]
    if encoded_input.numel() == 0:
        print(label)
```

What transformer version do you use?
Because execution of this line `if encoded_input.numel() == 0:` results in a following traceback

```python-traceback
KeyError: 'numel'

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
Input In [9], in <cell line: 1>()
      2 encoded_input = tokenizer(label, return_tensors="pt")
      3 input_ids = encoded_input["input_ids"]
----> 4 if encoded_input.numel() == 0:
      5     print(label)

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:249, in BatchEncoding.__getattr__(self, item)
    247     return self.data[item]
    248 except KeyError:
--> 249     raise AttributeError

AttributeError: 
```

I have tried so far 4.21.3 and 4.25.1

There was a typo:
```python
if encoded_input.numel() == 0:
```
should have been
```python
if input_ids.numel() == 0:
```

Sorry, for taking that long to run it. 
However, there was no label that got print

Okay, from the information I have, I cannot really see where the problem lies. If you can reduce the problem to something small, and without any external (data) dependencies, I can see whether I can reproduce the issue locally. Otherwise, you will have to dig into it yourself. I would suggest to use some debugger and set a breakpoint on the line where the exception is raised.

Well, I tried the same thing with open data, i.e. Nations dataset and the issue was exactly the same



On "nations", I could not reproduce the error, but experienced a different one during evaluation time. Maybe you could give #1209 a try?

Okay, I'll try and get back to you

I'll re-open this issue until we receive notification that this also solved the original issue.

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [19], in <cell line: 1>()
----> 1 result = pipeline(
      2     model=ERModel,
      3     training=training,
      4     testing=testing,
      5     validation=testing,
      6 #     training_loop='sLCWA',
      7 #     negative_sampler='basic',
      8     stopper='early',
      9     model_kwargs=dict(
     10         interaction="ermlpe",
     11         interaction_kwargs=dict(
     12             embedding_dim=entity_representations.shape[0],
     13         ),
     14         entity_representations=entity_representations,
     15         relation_representations_kwargs=dict(
     16             shape=entity_representations.shape,
     17         ),
     18     ),
     19     training_kwargs=dict(
     20         batch_size=32,
     21         num_epochs=1,
     22     ),
     23 #     evaluation_kwargs=dict(
     24 #         batch_size=1
     25 #     ),
     26     stopper_kwargs=dict(frequency=3, patience=3, relative_delta=0.0001),
     27     random_seed=42,
     28 #     optimizer='adamax',
     29 #     optimizer_kwargs=dict(lr=0.001),
     30 #     loss='softplus',
     31 #     negative_sampler_kwargs=dict(
     32 #         filtered=True,
     33 #         filterer='python-set',  
     34 #         num_negs_per_pos = 3, 
     35 #         corruption_scheme=(0)
     36 #     ),
     37 #     device='cpu',
     38 #     use_tqdm=True,
     39 )

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/pipeline/api.py:1562, in pipeline(dataset, dataset_kwargs, training, testing, validation, evaluation_entity_whitelist, evaluation_relation_whitelist, model, model_kwargs, interaction, interaction_kwargs, dimensions, loss, loss_kwargs, regularizer, regularizer_kwargs, optimizer, optimizer_kwargs, clear_optimizer, lr_scheduler, lr_scheduler_kwargs, training_loop, training_loop_kwargs, negative_sampler, negative_sampler_kwargs, epochs, training_kwargs, stopper, stopper_kwargs, evaluator, evaluator_kwargs, evaluation_kwargs, result_tracker, result_tracker_kwargs, metadata, device, random_seed, use_testing_data, evaluation_fallback, filter_validation_when_testing, use_tqdm)
   1539 evaluator_instance, evaluation_kwargs = _handle_evaluator(
   1540     _result_tracker=_result_tracker,
   1541     evaluator=evaluator,
   1542     evaluator_kwargs=evaluator_kwargs,
   1543     evaluation_kwargs=evaluation_kwargs,
   1544 )
   1546 stopper_instance, configuration, losses, train_seconds = _handle_training(
   1547     _result_tracker=_result_tracker,
   1548     training=training,
   (...)
   1559     use_tqdm=use_tqdm,
   1560 )
-> 1562 metric_results, evaluate_seconds = _handle_evaluation(
   1563     _result_tracker=_result_tracker,
   1564     model_instance=model_instance,
   1565     evaluator_instance=evaluator_instance,
   1566     stopper_instance=stopper_instance,
   1567     training=training,
   1568     testing=testing,
   1569     validation=validation,
   1570     training_kwargs=training_kwargs,
   1571     evaluation_kwargs=evaluation_kwargs,
   1572     use_testing_data=use_testing_data,
   1573     evaluation_fallback=evaluation_fallback,
   1574     filter_validation_when_testing=filter_validation_when_testing,
   1575     use_tqdm=use_tqdm,
   1576 )
   1577 _result_tracker.end_run()
   1579 return PipelineResult(
   1580     random_seed=_random_seed,
   1581     model=model_instance,
   (...)
   1590     evaluate_seconds=evaluate_seconds,
   1591 )

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/pipeline/api.py:1286, in _handle_evaluation(_result_tracker, model_instance, evaluator_instance, stopper_instance, training, testing, validation, training_kwargs, evaluation_kwargs, use_testing_data, evaluation_fallback, filter_validation_when_testing, use_tqdm)
   1277 _result_tracker.log_params(
   1278     params=dict(
   1279         evaluation_kwargs={
   (...)
   1283     )
   1284 )
   1285 evaluate_start_time = time.time()
-> 1286 metric_results: MetricResults = _safe_evaluate(
   1287     model=model_instance,
   1288     mapped_triples=mapped_triples,
   1289     evaluator=evaluator_instance,
   1290     evaluation_kwargs=evaluation_kwargs,
   1291     evaluation_fallback=evaluation_fallback,
   1292 )
   1293 evaluate_end_time = time.time() - evaluate_start_time
   1294 step = training_kwargs.get("num_epochs")

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/pipeline/api.py:1619, in _safe_evaluate(model, mapped_triples, evaluator, evaluation_kwargs, evaluation_fallback)
   1617 while True:
   1618     try:
-> 1619         metric_results: MetricResults = evaluator.evaluate(
   1620             model=model,
   1621             mapped_triples=mapped_triples,
   1622             **evaluation_kwargs,
   1623         )
   1624     except (MemoryError, RuntimeError) as e:
   1625         # If the evaluation still fail using the CPU, the error is raised
   1626         if model.device.type != "cuda" or not evaluation_fallback:

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:200, in Evaluator.evaluate(self, model, mapped_triples, batch_size, slice_size, **kwargs)
    195     logger.info(
    196         "Currently automatic memory optimization only supports GPUs, but you're using a CPU. "
    197         "Therefore, the batch_size will be set to the default value.",
    198     )
    199 else:
--> 200     batch_size, slice_size = self.batch_and_slice(
    201         model=model,
    202         mapped_triples=mapped_triples,
    203         batch_size=batch_size,
    204         **kwargs,
    205     )
    206     # The batch_size and slice_size should be accessible to outside objects for re-use, e.g. early stoppers.
    207     self.batch_size = batch_size

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:273, in Evaluator.batch_and_slice(self, model, mapped_triples, batch_size, **kwargs)
    270 # We need to try slicing, if the evaluation for the batch_size search never succeeded
    271 # we do not need to repeat time-consuming checks
    272 kwargs["do_time_consuming_checks"] = False
--> 273 slice_size, evaluated_once = self._param_size_search(
    274     key="slice_size",
    275     # infer start value
    276     start_value=None,
    277     model=model,
    278     mapped_triples=mapped_triples,
    279     **kwargs,
    280 )
    281 if not evaluated_once:
    282     raise MemoryError("The current model can't be trained on this hardware with these parameters.")

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:334, in Evaluator._param_size_search(self, key, start_value, model, mapped_triples, **kwargs)
    332     gc.collect()
    333     torch.cuda.empty_cache()
--> 334     evaluate(
    335         model=model,
    336         mapped_triples=mapped_triples,
    337         evaluator=self,
    338         only_size_probing=True,
    339         batch_size=values_dict.get("batch_size"),
    340         slice_size=values_dict.get("slice_size"),
    341         **kwargs,
    342     )
    343     evaluated_once = True
    344 except RuntimeError as runtime_error:
    345     # Due to the caused OOM Runtime Error, the failed model has to be cleared to avoid memory leakage

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:687, in evaluate(model, mapped_triples, evaluator, only_size_probing, batch_size, slice_size, device, use_tqdm, tqdm_kwargs, restrict_entities_to, restrict_relations_to, do_time_consuming_checks, additional_filter_triples, pre_filtered_triples, targets, mode)
    685 relation_filter = None
    686 for target in targets:
--> 687     relation_filter = _evaluate_batch(
    688         batch=batch,
    689         model=model,
    690         target=target,
    691         evaluator=evaluator,
    692         slice_size=slice_size,
    693         all_pos_triples=all_pos_triples,
    694         relation_filter=relation_filter,
    695         restrict_entities_to=restrict_entities_to,
    696         mode=mode,
    697     )
    699 # If we only probe sizes we do not need more than one batch
    700 if only_size_probing and evaluated_once:

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/evaluation/evaluator.py:760, in _evaluate_batch(batch, model, target, evaluator, slice_size, all_pos_triples, relation_filter, restrict_entities_to, mode)
    720 def _evaluate_batch(
    721     batch: MappedTriples,
    722     model: Model,
   (...)
    730     mode: Optional[InductiveMode],
    731 ) -> torch.BoolTensor:
    732     """
    733     Evaluate ranking for batch.
    734 
   (...)
    758         The relation filter, which can be re-used for the same batch.
    759     """
--> 760     scores = model.predict(hrt_batch=batch, target=target, slice_size=slice_size, mode=mode)
    762     if evaluator.filtered or evaluator.requires_positive_mask:
    763         column = TARGET_TO_INDEX[target]

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/models/base.py:481, in Model.predict(self, hrt_batch, target, full_batch, ids, **kwargs)
    479     if full_batch:
    480         hrt_batch = hrt_batch[:, 1:3]
--> 481     return self.predict_h(hrt_batch, **kwargs, heads=ids)
    483 raise ValueError(f"Unknown target={target}")

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/models/base.py:374, in Model.predict_h(self, rt_batch, **kwargs)
    372     scores = self.score_h_inverse(rt_batch=rt_batch, **kwargs)
    373 else:
--> 374     scores = self.score_h(rt_batch, **kwargs)
    375 if self.predict_with_sigmoid:
    376     scores = torch.sigmoid(scores)

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/models/nbase.py:517, in ERModel.score_h(self, rt_batch, slice_size, mode, heads)
    509 def score_h(
    510     self,
    511     rt_batch: torch.LongTensor,
   (...)
    515     heads: Optional[torch.LongTensor] = None,
    516 ) -> torch.FloatTensor:  # noqa: D102
--> 517     self._check_slicing(slice_size=slice_size)
    518     # add broadcast dimension
    519     rt_batch = rt_batch.unsqueeze(dim=1)

File /home/user/conda/envs/test_env/lib/python3.8/site-packages/pykeen/models/nbase.py:484, in ERModel._check_slicing(self, slice_size)
    482     return
    483 if get_batchnorm_modules(self):  # if there are any, this is truthy
--> 484     raise ValueError("This model does not support slicing, since it has batch normalization layers.")

ValueError: This model does not support slicing, since it has batch normalization layers.
```

The error is definitely different not but I still do not understand what to do with that

Okay, seems like the training succeeded, but during evaluation an error occurred. You can see that from this part of the traceback
```
   1546 stopper_instance, configuration, losses, train_seconds = _handle_training(
   1547     _result_tracker=_result_tracker,
   1548     training=training,
   (...)
   1559     use_tqdm=use_tqdm,
   1560 )
-> 1562 metric_results, evaluate_seconds = _handle_evaluation(
```

By default, knowledge graph embedding models are evaluated with 1:n scoring (cf. https://pykeen.readthedocs.io/en/stable/tutorial/understanding_evaluation.html), i.e., for a given evaluation triple `(h, r, t)` we need to calculate scores for *all* possible tail entities `t'`. For efficiency, this is usually done in one forward pass. However, depending on models and/or interactions this may require a large amount of (GPU) memory. In that case, there are some fallback methods (described here: https://pykeen.readthedocs.io/en/stable/tutorial/performance.html#sub-batching-slicing). The last option is what is called `slicing`, i.e., we have already reduced the evaluation batch size to 1, but still see ouf-of-memory issues. In that case, we would then make "slices" of the entities to score, e.g., first calculate scores for the first 10 entities, then for the next ones, etc. However, for some models, in particular those which have layers where the output for one sample depends on the batch, could give different results depending on the chosen slices. That is why an error is raised instead.

Now in your case, I am not sure why we need to raise that, since the *evaluation* behaviour of batch normalization layers does *not* depend on other batch members (while its training behaviour does). So I think we could relax this check.


> Now in your case, I am not sure why we need to raise that, since the evaluation behaviour of batch normalization layers does not depend on other batch members (while its training behaviour does). So I think we could relax this check.

#1215 is now merged to `master`. If you want, you can try whether this fixes your problem. To do so, you'll need to install the `master` version directly from git:
```console
pip install -U git+https://github.com/pykeen/pykeen.git
```

I tried it, however there are now cuda errors but my evaluation batch size is 1 and I have 40 GB of GPU memory
```
RuntimeError: CUDA out of memory. Tried to allocate 17.43 GiB (GPU 0; 39.59 GiB total capacity; 28.53 GiB already allocated; 9.13 GiB free; 28.57 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
```

Hello, I have similar problem on lastest version and _bert-base-multilingual-uncased_. After 9 epoch of training I got this on evaluation.

```python-traceback
File ~/.conda/envs/kg/lib/python3.8/site-packages/torch/nn/functional.py:2210, in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2204     # Note [embedding_renorm set_grad_enabled]
   2205     # XXX: equivalent to
   2206     # with torch.no_grad():
   2207     #   torch.embedding_renorm_
   2208     # remove once script supports set_grad_enabled
   2209     _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2210 return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)

OutOfMemoryError: CUDA out of memory. Tried to allocate 323.64 GiB (GPU 0; 39.59 GiB total capacity; 4.87 GiB already allocated; 33.31 GiB free; 4.99 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
```

> 

| Key             | Value                    |
|-----------------|--------------------------|
| OS              | posix                    |
| Platform        | Linux                    |
| Release         | 5.4.0-122-generic        |
| Time            | Mon Feb  6 15:51:43 2023 |
| Python          | 3.8.16                   |
| PyKEEN          | 1.10.1-dev               |
| PyKEEN Hash     | UNHASHED                 |
| PyKEEN Branch   |                          |
| PyTorch         | 1.13.1+cu117             |
| CUDA Available? | true                     |
| CUDA Version    | 11.7                     |
| cuDNN Version   | 8500                     |