Repository: huggingface/transformers

Issue:
[BlenderbotSmallTokenizer] Cannot download tokenizer
When running: 

```python
from transformers import BlenderbotSmallTokenizer
tok = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot_small-90M")
```

the command fails with the error

```~/python_bin/transformers/tokenization_utils_base.py in _from_pretrained(cls, resolved_vocab_files, pretrained_model_name_or_path, init_configuration, *init_inputs, **kwargs)
   1894         # Instantiate tokenizer.
   1895         try:
-> 1896             tokenizer = cls(*init_inputs, **init_kwargs)
   1897         except OSError:
   1898             raise OSError(

~/python_bin/transformers/models/blenderbot_small/tokenization_blenderbot_small.py in __init__(self, vocab_file, merges_file, bos_token, eos_token, unk_token, pad_token, **kwargs)
    107 
    108         with open(vocab_file, encoding="utf-8") as vocab_handle:
--> 109             self.encoder = json.load(vocab_handle)
    110         self.decoder = {v: k for k, v in self.encoder.items()}
    111         with open(merges_file, encoding="utf-8") as merges_handle:

/usr/lib/python3.7/json/__init__.py in load(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    294         cls=cls, object_hook=object_hook,
    295         parse_float=parse_float, parse_int=parse_int,
--> 296         parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
    297 
    298 

/usr/lib/python3.7/json/__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    346             parse_int is None and parse_float is None and
    347             parse_constant is None and object_pairs_hook is None and not kw):
--> 348         return _default_decoder.decode(s)
    349     if cls is None:
    350         cls = JSONDecoder

/usr/lib/python3.7/json/decoder.py in decode(self, s, _w)
    335 
    336         """
--> 337         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
    338         end = _w(s, end).end()
    339         if end != len(s):

/usr/lib/python3.7/json/decoder.py in raw_decode(self, s, idx)
    353             obj, end = self.scan_once(s, idx)
    354         except StopIteration as err:
--> 355             raise JSONDecodeError("Expecting value", s, err.value) from None
    356         return obj, end

JSONDecodeError: Expecting value: line 1 column 1 (char 0)
```

This is strange since `"facebook/blenderbot_small-90M"` is just a copy of `"facebook/blenderbot-90M"` which works:

```python
from transformers import BlenderbotSmallTokenizer
tok = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M")
```


Discussion:
Is `blenderbot-small` a valid model_type?

Or maybe this file https://huggingface.co/facebook/blenderbot_small-90M/blob/main/tokenizer_config.json is an issue?
yeah, `blenderbot-small` is valid. One can download both the model and the config correctly:

```python
from transformers import BlenderbotSmallModel

model = BlenderbotSmallModel.from_pretrained("facebook/blenderbot_small-90M")
```

I think it has something to do with the tokenizers `vocab.json` file. But it's 1-to-1 the same file as in `"facebook/blenderbot-90M"` which can correctly be loaded...I'll have to check in more detail in the next days. There is probably a problem with the BlenderbotSmallTokenizer

Resolving patch:
diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot.py b/src/transformers/models/blenderbot/tokenization_blenderbot.py
--- a/src/transformers/models/blenderbot/tokenization_blenderbot.py
+++ b/src/transformers/models/blenderbot/tokenization_blenderbot.py
@@ -48,9 +48,11 @@ class BlenderbotTokenizer(RobertaTokenizer):
         "tokenizer_config_file": "tokenizer_config.json",
     }
     pretrained_vocab_files_map = {
-        "vocab_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/vocab.json"},
-        "merges_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/merges.txt"},
-        "tokenizer_config_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/tokenizer_config.json"},
+        "vocab_file": {CKPT_3B: "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
+        "merges_file": {CKPT_3B: "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
+        "tokenizer_config_file": {
+            CKPT_3B: "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
+        },
     }
     max_model_input_sizes = {"facebook/blenderbot-3B": 128}
 
diff --git a/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py b/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
--- a/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
+++ b/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
@@ -82,13 +82,13 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
     }
     pretrained_vocab_files_map = {
         "vocab_file": {
-            "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/blob/main/vocab.json"
+            "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"
         },
         "merges_file": {
-            "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/blob/main/merges.txt"
+            "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"
         },
         "tokenizer_config_file": {
-            "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/blob/main/tokenizer.json"
+            "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer.json"
         },
     }
     max_model_input_sizes = {"facebook/blenderbot_small-90M": 512}


Test patch: