thefrigidliquidation commited on
Commit
d648763
1 Parent(s): 5034acf

Remove usage of patched tokenizer for training

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -19,6 +19,6 @@
19
  "special_tokens_map_file": null,
20
  "src_lang": "jpn_Jpan",
21
  "tgt_lang": "eng_Latn",
22
- "tokenizer_class": "TrainingNllbTokenizer",
23
  "unk_token": "<unk>"
24
  }
 
19
  "special_tokens_map_file": null,
20
  "src_lang": "jpn_Jpan",
21
  "tgt_lang": "eng_Latn",
22
+ "tokenizer_class": "NllbTokenizer",
23
  "unk_token": "<unk>"
24
  }