Fix unsupported operand type(s) for |
Browse files
src/axolotl/prompt_tokenizers.py
CHANGED
@@ -4,7 +4,7 @@ import abc
|
|
4 |
import copy
|
5 |
import functools
|
6 |
import logging
|
7 |
-
from typing import Dict, List, Tuple
|
8 |
|
9 |
from transformers import PreTrainedTokenizer
|
10 |
|
@@ -433,7 +433,7 @@ def parse_tokenized_to_result(
|
|
433 |
current_len: int,
|
434 |
res: Dict[str, List[int]],
|
435 |
labels: list[int],
|
436 |
-
pad_token_id: int
|
437 |
) -> Tuple[Dict[str, List[int]], int]:
|
438 |
"""
|
439 |
Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
|
|
|
4 |
import copy
|
5 |
import functools
|
6 |
import logging
|
7 |
+
from typing import Dict, List, Tuple, Union
|
8 |
|
9 |
from transformers import PreTrainedTokenizer
|
10 |
|
|
|
433 |
current_len: int,
|
434 |
res: Dict[str, List[int]],
|
435 |
labels: list[int],
|
436 |
+
pad_token_id: Union[int, None] = None,
|
437 |
) -> Tuple[Dict[str, List[int]], int]:
|
438 |
"""
|
439 |
Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
|