Can't find vocabulary file or is corrupted for MarianTokenizer
Author: sadeqa
Pull request: no

# 🐛 Bug

## Information

Model I am using MarianMT:

The problem arises when using the tokenizer with from_pretrained

## To reproduce

from transformers import MarianTokenizer, MarianMTModel
src = 'fr'  # source language
trg = 'en'  # target language
sample_text = "où est l'arrêt de bus ?"
mname = f'Helsinki-NLP/opus-mt-{src}-{trg}'
model = MarianMTModel.from_pretrained(mname)
tok = MarianTokenizer.from_pretrained(mname)

stdbuf was not found; communication with perl may hang due to stdio buffering.
FileNotFoundError                         Traceback (most recent call last)
~\Anaconda3\envs\pytorch\lib\site-packages\transformers\tokenization_utils.py in _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs)
   1054         try:
-> 1055             tokenizer = cls(*init_inputs, **init_kwargs)
   1056         except OSError:

~\Anaconda3\envs\pytorch\lib\site-packages\transformers\tokenization_marian.py in __init__(self, vocab, source_spm, target_spm, source_lang, target_lang, unk_token, eos_token, pad_token, max_len)
     88 
---> 89             self.punc_normalizer = MosesPunctuationNormalizer(source_lang)
     90         except ImportError:

~\Anaconda3\envs\pytorch\lib\site-packages\mosestokenizer\punctnormalizer.py in __init__(self, lang)
     46         argv = ["perl", program, "-b", "-l", self.lang]
---> 47         super().__init__(argv)
     48 

~\Anaconda3\envs\pytorch\lib\site-packages\toolwrapper.py in __init__(self, argv, encoding, start, cwd, stdbuf, stderr, env)
     63         if start:
---> 64             self.start()
     65 

~\Anaconda3\envs\pytorch\lib\site-packages\toolwrapper.py in start(self)
    107             cwd=self.cwd,
--> 108             env=env,
    109         )

~\Anaconda3\envs\pytorch\lib\subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    799                                 errread, errwrite,
--> 800                                 restore_signals, start_new_session)
    801         except:

~\Anaconda3\envs\pytorch\lib\subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)
   1206                                          os.fspath(cwd) if cwd is not None else None,
-> 1207                                          startupinfo)
   1208             finally:

FileNotFoundError: [WinError 2] The system cannot find the file specified

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
<ipython-input-10-56dab20251f1> in <module>
      6 mname = f'Helsinki-NLP/opus-mt-{src}-{trg}'
      7 model = MarianMTModel.from_pretrained(mname)
----> 8 tok = MarianTokenizer.from_pretrained(mname)

~\Anaconda3\envs\pytorch\lib\site-packages\transformers\tokenization_utils.py in from_pretrained(cls, *inputs, **kwargs)
    900 
    901         """
--> 902         return cls._from_pretrained(*inputs, **kwargs)
    903 
    904     @classmethod

~\Anaconda3\envs\pytorch\lib\site-packages\transformers\tokenization_utils.py in _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs)
   1056         except OSError:
   1057             raise OSError(
-> 1058                 "Unable to load vocabulary from file. "
   1059                 "Please check that the provided vocabulary is accessible and not corrupted."
   1060             )

OSError: Unable to load vocabulary from file. Please check that the provided vocabulary is accessible and not corrupted.

----------------------------------------
I'm working on a windows 10 environment


Comments:
Closing in favour of the better formulated question here: https://github.com/huggingface/transformers/issues/4491

Can anyone help with this issue: #5040 ?