SaulLu commited on
Commit
126f3f6
1 Parent(s): 898677c

Add tokenizer type in config.json

Browse files
Files changed (1) hide show
  1. config.json +2 -1
config.json CHANGED
@@ -49,5 +49,6 @@
49
  "position_embedding_type": "absolute",
50
  "transformers_version": "4.6.1",
51
  "type_vocab_size": 2,
52
- "vocab_size": 32000
 
53
  }
 
49
  "position_embedding_type": "absolute",
50
  "transformers_version": "4.6.1",
51
  "type_vocab_size": 2,
52
+ "vocab_size": 32000,
53
+ "tokenizer_class": "PreTrainedTokenizerFast"
54
  }