Update tokenizer.py
Browse files- tokenizer.py +1 -0
tokenizer.py
CHANGED
|
@@ -6,6 +6,7 @@ import pathlib
|
|
| 6 |
import torch
|
| 7 |
import tqdm
|
| 8 |
from tokenizers import Tokenizer
|
|
|
|
| 9 |
from abc import abstractmethod
|
| 10 |
from typing import Any, List, Union
|
| 11 |
import numpy as np
|
|
|
|
| 6 |
import torch
|
| 7 |
import tqdm
|
| 8 |
from tokenizers import Tokenizer
|
| 9 |
+
from transformers.tokenization_utils import PreTrainedTokenizer
|
| 10 |
from abc import abstractmethod
|
| 11 |
from typing import Any, List, Union
|
| 12 |
import numpy as np
|