codebyzeb commited on
Commit
92bfd8b
1 Parent(s): eaf3e8d

Upload tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +6 -0
  2. tokenizer.json +126 -0
  3. tokenizer_config.json +8 -0
  4. vocab.json +1 -0
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "BOS",
3
+ "eos_token": "EOS",
4
+ "pad_token": "PAD",
5
+ "unk_token": "UNK"
6
+ }
tokenizer.json ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "UNK",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "PAD",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "BOS",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "EOS",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ }
42
+ ],
43
+ "normalizer": {
44
+ "type": "Sequence",
45
+ "normalizers": [
46
+ {
47
+ "type": "Replace",
48
+ "pattern": {
49
+ "String": "\n"
50
+ },
51
+ "content": " UTT_BOUNDARY"
52
+ },
53
+ {
54
+ "type": "Strip",
55
+ "strip_left": true,
56
+ "strip_right": true
57
+ }
58
+ ]
59
+ },
60
+ "pre_tokenizer": {
61
+ "type": "Whitespace"
62
+ },
63
+ "post_processor": null,
64
+ "decoder": null,
65
+ "model": {
66
+ "type": "WordLevel",
67
+ "vocab": {
68
+ "UNK": 0,
69
+ "PAD": 1,
70
+ "BOS": 2,
71
+ "EOS": 3,
72
+ "WORD_BOUNDARY": 4,
73
+ "UTT_BOUNDARY": 5,
74
+ "m": 6,
75
+ "ɛ": 7,
76
+ "ɲ": 8,
77
+ "y": 9,
78
+ "n": 10,
79
+ "k": 11,
80
+ "ɑ": 12,
81
+ "r": 13,
82
+ "aː": 14,
83
+ "d": 15,
84
+ "i": 16,
85
+ "o": 17,
86
+ "h": 18,
87
+ "z": 19,
88
+ "v": 20,
89
+ "l": 21,
90
+ "eː": 22,
91
+ "j": 23,
92
+ "ʃ": 24,
93
+ "ɟ": 25,
94
+ "s": 26,
95
+ "oː": 27,
96
+ "p": 28,
97
+ "t": 29,
98
+ "tsː": 30,
99
+ "b": 31,
100
+ "u": 32,
101
+ "ɡ": 33,
102
+ "tː": 34,
103
+ "f": 35,
104
+ "ø": 36,
105
+ "t̠ʃ": 37,
106
+ "uː": 38,
107
+ "iː": 39,
108
+ "ts": 40,
109
+ "ɟː": 41,
110
+ "yː": 42,
111
+ "øː": 43,
112
+ "ʎ": 44,
113
+ "t̠ʃː": 45,
114
+ "c": 46,
115
+ "ɡː": 47,
116
+ "kː": 48,
117
+ "ɑː": 49,
118
+ "dː": 50,
119
+ "pː": 51,
120
+ "ʒ": 52,
121
+ "cː": 53,
122
+ "bː": 54
123
+ },
124
+ "unk_token": "UNK"
125
+ }
126
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "model_max_length": 1000000000000000019884624838656,
6
+ "tokenizer_class": "GPT2Tokenizer",
7
+ "unk_token": "<|endoftext|>"
8
+ }
vocab.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"UNK":0,"PAD":1,"BOS":2,"EOS":3,"WORD_BOUNDARY":4,"UTT_BOUNDARY":5,"m":6,"ɛ":7,"ɲ":8,"y":9,"n":10,"k":11,"ɑ":12,"r":13,"aː":14,"d":15,"i":16,"o":17,"h":18,"z":19,"v":20,"l":21,"eː":22,"j":23,"ʃ":24,"ɟ":25,"s":26,"oː":27,"p":28,"t":29,"tsː":30,"b":31,"u":32,"ɡ":33,"tː":34,"f":35,"ø":36,"t̠ʃ":37,"uː":38,"iː":39,"ts":40,"ɟː":41,"yː":42,"øː":43,"ʎ":44,"t̠ʃː":45,"c":46,"ɡː":47,"kː":48,"ɑː":49,"dː":50,"pː":51,"ʒ":52,"cː":53,"bː":54}