Hugging Face's logo

Skier8402
/
XLNetTokenizerFast_unigram_like

English
NLP
Unigram
Model card Files Community
Skier8402 commited on
Commit
b46df25
·
verified ·
1 Parent(s): f75a8c4

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +9 -0
  2. tokenizer.json +126 -0
  3. tokenizer_config.json +71 -0
special_tokens_map.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<cls>",
4
+ "eos_token": "</s>",
5
+ "mask_token": "<mask>",
6
+ "pad_token": "<pad>",
7
+ "sep_token": "<sep>",
8
+ "unk_token": "<unk>"
9
+ }
tokenizer.json ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<unk>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<cls>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "<sep>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<pad>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "<mask>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 5,
53
+ "content": "<s>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ },
60
+ {
61
+ "id": 6,
62
+ "content": "</s>",
63
+ "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
+ }
69
+ ],
70
+ "normalizer": {
71
+ "type": "Sequence",
72
+ "normalizers": [
73
+ {
74
+ "type": "Replace",
75
+ "pattern": {
76
+ "String": "``"
77
+ },
78
+ "content": "\""
79
+ },
80
+ {
81
+ "type": "Replace",
82
+ "pattern": {
83
+ "String": "''"
84
+ },
85
+ "content": "\""
86
+ },
87
+ {
88
+ "type": "NFKD"
89
+ },
90
+ {
91
+ "type": "StripAccents"
92
+ },
93
+ {
94
+ "type": "Replace",
95
+ "pattern": {
96
+ "Regex": " {2,}"
97
+ },
98
+ "content": " "
99
+ }
100
+ ]
101
+ },
102
+ "pre_tokenizer": {
103
+ "type": "Metaspace",
104
+ "replacement": "▁",
105
+ "add_prefix_space": true,
106
+ "prepend_scheme": "always"
107
+ },
108
+ "post_processor": null,
109
+ "decoder": {
110
+ "type": "Metaspace",
111
+ "replacement": "▁",
112
+ "add_prefix_space": true,
113
+ "prepend_scheme": "always"
114
+ },
115
+ "model": {
116
+ "type": "Unigram",
117
+ "unk_id": 0,
118
+ "vocab": [
119
+ [
120
+ "<unk>",
121
+ 0.0
122
+ ]
123
+ ],
124
+ "byte_fallback": false
125
+ }
126
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<cls>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<sep>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<pad>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "<s>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "6": {
52
+ "content": "</s>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ }
59
+ },
60
+ "bos_token": "<s>",
61
+ "clean_up_tokenization_spaces": true,
62
+ "cls_token": "<cls>",
63
+ "eos_token": "</s>",
64
+ "mask_token": "<mask>",
65
+ "model_max_length": 1000000000000000019884624838656,
66
+ "pad_token": "<pad>",
67
+ "padding_side": "left",
68
+ "sep_token": "<sep>",
69
+ "tokenizer_class": "PreTrainedTokenizerFast",
70
+ "unk_token": "<unk>"
71
+ }