Update tokenization_indictrans.py
Browse files
tokenization_indictrans.py
CHANGED
|
@@ -60,6 +60,7 @@ class IndicTransTokenizer(PreTrainedTokenizer):
|
|
| 60 |
_added_tokens_encoder = {}
|
| 61 |
_added_tokens_decoder = {}
|
| 62 |
|
|
|
|
| 63 |
model_input_names = ["input_ids", "attention_mask"]
|
| 64 |
|
| 65 |
def __init__(
|
|
|
|
| 60 |
_added_tokens_encoder = {}
|
| 61 |
_added_tokens_decoder = {}
|
| 62 |
|
| 63 |
+
vocab_files_names = VOCAB_FILES_NAMES
|
| 64 |
model_input_names = ["input_ids", "attention_mask"]
|
| 65 |
|
| 66 |
def __init__(
|