avans06 commited on
Commit
c78307c
1 Parent(s): 6b56c54

According to the recommendation of transformers,

Browse files
Files changed (1) hide show
  1. src/nllb/nllbModel.py +1 -1
src/nllb/nllbModel.py CHANGED
@@ -82,7 +82,7 @@ class NllbModel:
82
  self.trans_model = ctranslate2.Translator(self.model_path, compute_type="auto", device=self.device)
83
  elif "mt5" in self.model_path:
84
  self.mt5_prefix = self.whisper_lang.code + "2" + self.nllb_lang.code_whisper + ": "
85
- self.trans_tokenizer = transformers.T5Tokenizer.from_pretrained(self.model_path) #requires spiece.model
86
  self.trans_model = transformers.MT5ForConditionalGeneration.from_pretrained(self.model_path)
87
  self.trans_translator = transformers.pipeline('text2text-generation', model=self.trans_model, device=self.device, tokenizer=self.trans_tokenizer)
88
  else: #NLLB
 
82
  self.trans_model = ctranslate2.Translator(self.model_path, compute_type="auto", device=self.device)
83
  elif "mt5" in self.model_path:
84
  self.mt5_prefix = self.whisper_lang.code + "2" + self.nllb_lang.code_whisper + ": "
85
+ self.trans_tokenizer = transformers.T5Tokenizer.from_pretrained(self.model_path, legacy=False) #requires spiece.model
86
  self.trans_model = transformers.MT5ForConditionalGeneration.from_pretrained(self.model_path)
87
  self.trans_translator = transformers.pipeline('text2text-generation', model=self.trans_model, device=self.device, tokenizer=self.trans_tokenizer)
88
  else: #NLLB