chiragjn commited on
Commit
4fd8c92
1 Parent(s): 1e7fdcc

Add `tokenizer_class` to get `pipeline` to load tokenizer

Browse files
Files changed (1) hide show
  1. config.json +2 -1
config.json CHANGED
@@ -24,5 +24,6 @@
24
  "torch_dtype": "bfloat16",
25
  "transformers_version": "4.26.0",
26
  "use_cache": true,
27
- "vocab_size": 65024
 
28
  }
 
24
  "torch_dtype": "bfloat16",
25
  "transformers_version": "4.26.0",
26
  "use_cache": true,
27
+ "vocab_size": 65024,
28
+ "tokenizer_class": "PreTrainedTokenizerFast"
29
  }