Hugging Face's logo

WikiQuality
/
pre_filtered.ln

Fill-Mask
Transformers
Safetensors
deberta
Inference Endpoints
Model card Files Community
kushaltatariya commited on
Commit
9648065
•
1 Parent(s): 815af9a

Upload tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +7 -5
  2. tokenization.model +2 -2
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +31 -11
special_tokens_map.json CHANGED
@@ -1,7 +1,9 @@
1
  {
2
- "bos_token": "<s>",
3
- "eos_token": "</s>",
4
- "mask_token": "<mask>",
5
- "pad_token": "<pad>",
6
- "unk_token": "<unk>"
 
 
7
  }
 
1
  {
2
+ "bos_token": "[BOS]",
3
+ "cls_token": "[CLS]",
4
+ "eos_token": "[EOS]",
5
+ "mask_token": "[MASK]",
6
+ "pad_token": "[PAD]",
7
+ "sep_token": "[SEP]",
8
+ "unk_token": "[UNK]"
9
  }
tokenization.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:14c97d6c1c99a660fd45c77db1e97f4860453448fa62e65e61c73081f705a5e3
3
- size 261340
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c0fc51c0ad6ca0090025157807fef40ab2740ca79ca0fef7af19d5c45dee273
3
+ size 261013
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -4,7 +4,7 @@
4
  "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
  "0": {
7
- "content": "<pad>",
8
  "lstrip": false,
9
  "normalized": false,
10
  "rstrip": false,
@@ -12,7 +12,7 @@
12
  "special": true
13
  },
14
  "1": {
15
- "content": "<unk>",
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
@@ -20,7 +20,7 @@
20
  "special": true
21
  },
22
  "2": {
23
- "content": "<s>",
24
  "lstrip": false,
25
  "normalized": false,
26
  "rstrip": false,
@@ -28,7 +28,7 @@
28
  "special": true
29
  },
30
  "3": {
31
- "content": "</s>",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
@@ -36,7 +36,23 @@
36
  "special": true
37
  },
38
  "4": {
39
- "content": "<mask>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  "lstrip": false,
41
  "normalized": false,
42
  "rstrip": false,
@@ -44,20 +60,24 @@
44
  "special": false
45
  }
46
  },
47
- "bos_token": "<s>",
48
  "bos_token_id": 2,
49
  "clean_up_tokenization_spaces": false,
50
- "eos_token": "</s>",
 
 
51
  "eos_token_id": 3,
52
  "legacy": true,
53
- "mask_token": "<mask>",
54
- "mask_token_id": 4,
55
  "model_max_length": 1000000000000000019884624838656,
56
- "pad_token": "<pad>",
57
  "pad_token_id": 0,
 
 
58
  "sp_model_kwargs": {},
59
  "spaces_between_special_tokens": false,
60
  "tokenizer_class": "HfSentencePieceTokenizer",
61
- "unk_token": "<unk>",
62
  "unk_token_id": 1
63
  }
 
4
  "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
  "0": {
7
+ "content": "[PAD]",
8
  "lstrip": false,
9
  "normalized": false,
10
  "rstrip": false,
 
12
  "special": true
13
  },
14
  "1": {
15
+ "content": "[UNK]",
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
 
20
  "special": true
21
  },
22
  "2": {
23
+ "content": "[BOS]",
24
  "lstrip": false,
25
  "normalized": false,
26
  "rstrip": false,
 
28
  "special": true
29
  },
30
  "3": {
31
+ "content": "[EOS]",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
 
36
  "special": true
37
  },
38
  "4": {
39
+ "content": "[CLS]",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "5": {
47
+ "content": "[SEP]",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": false,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "6": {
55
+ "content": "[MASK]",
56
  "lstrip": false,
57
  "normalized": false,
58
  "rstrip": false,
 
60
  "special": false
61
  }
62
  },
63
+ "bos_token": "[BOS]",
64
  "bos_token_id": 2,
65
  "clean_up_tokenization_spaces": false,
66
+ "cls_token": "[CLS]",
67
+ "cls_token_id": 4,
68
+ "eos_token": "[EOS]",
69
  "eos_token_id": 3,
70
  "legacy": true,
71
+ "mask_token": "[MASK]",
72
+ "mask_token_id": 6,
73
  "model_max_length": 1000000000000000019884624838656,
74
+ "pad_token": "[PAD]",
75
  "pad_token_id": 0,
76
+ "sep_token": "[SEP]",
77
+ "sep_token_id": 5,
78
  "sp_model_kwargs": {},
79
  "spaces_between_special_tokens": false,
80
  "tokenizer_class": "HfSentencePieceTokenizer",
81
+ "unk_token": "[UNK]",
82
  "unk_token_id": 1
83
  }