Hugging Face's logo

lokinfey
/
Phi-4-ov-int4

OpenVINO
phi3
custom_code
Model card Files Community
lokinfey commited on
Commit
70a8ef4
•
1 Parent(s): 4a0b647

Upload 12 files

Browse files
Files changed (12) hide show
  1. config.json +33 -0
  2. generation_config.json +9 -0
  3. merges.txt +0 -0
  4. openvino_detokenizer.bin +3 -0
  5. openvino_detokenizer.xml +183 -0
  6. openvino_model.xml +0 -0
  7. openvino_tokenizer.bin +3 -0
  8. openvino_tokenizer.xml +701 -0
  9. special_tokens_map.json +24 -0
  10. tokenizer.json +0 -0
  11. tokenizer_config.json +781 -0
  12. vocab.json +0 -0
config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": ".\\phi-4\\",
3
+ "architectures": [
4
+ "Phi3ForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "auto_map": {},
9
+ "bos_token_id": 100257,
10
+ "embd_pdrop": 0.0,
11
+ "eos_token_id": 100257,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 5120,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 17920,
16
+ "max_position_embeddings": 16384,
17
+ "model_type": "phi3",
18
+ "num_attention_heads": 40,
19
+ "num_hidden_layers": 40,
20
+ "num_key_value_heads": 10,
21
+ "original_max_position_embeddings": 16384,
22
+ "pad_token_id": 100257,
23
+ "resid_pdrop": 0.0,
24
+ "rms_norm_eps": 1e-05,
25
+ "rope_scaling": null,
26
+ "rope_theta": 250000,
27
+ "sliding_window": null,
28
+ "tie_word_embeddings": false,
29
+ "torch_dtype": "bfloat16",
30
+ "transformers_version": "4.44.2",
31
+ "use_cache": true,
32
+ "vocab_size": 100352
33
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 100257,
4
+ "eos_token_id": [
5
+ 100257,
6
+ 100265
7
+ ],
8
+ "transformers_version": "4.44.2"
9
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
openvino_detokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06abbcf3af8ca5b3bca4247f9b75174befbfb10655b59457c1a4d086269c0910
3
+ size 1046586
openvino_detokenizer.xml ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="detokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_180292" type="Parameter" version="opset1">
5
+ <data shape="?,?" element_type="i64" />
6
+ <output>
7
+ <port id="0" precision="I64" names="Parameter_180292">
8
+ <dim>-1</dim>
9
+ <dim>-1</dim>
10
+ </port>
11
+ </output>
12
+ </layer>
13
+ <layer id="1" name="Convert_180303" type="Convert" version="opset1">
14
+ <data destination_type="i32" />
15
+ <input>
16
+ <port id="0" precision="I64">
17
+ <dim>-1</dim>
18
+ <dim>-1</dim>
19
+ </port>
20
+ </input>
21
+ <output>
22
+ <port id="1" precision="I32">
23
+ <dim>-1</dim>
24
+ <dim>-1</dim>
25
+ </port>
26
+ </output>
27
+ </layer>
28
+ <layer id="2" name="Constant_180267" type="Const" version="opset1">
29
+ <data element_type="u8" shape="1046586" offset="0" size="1046586" />
30
+ <output>
31
+ <port id="0" precision="U8">
32
+ <dim>1046586</dim>
33
+ </port>
34
+ </output>
35
+ </layer>
36
+ <layer id="3" name="StringTensorUnpack_180268" type="StringTensorUnpack" version="extension">
37
+ <data mode="begins_ends" />
38
+ <input>
39
+ <port id="0" precision="U8">
40
+ <dim>1046586</dim>
41
+ </port>
42
+ </input>
43
+ <output>
44
+ <port id="1" precision="I32">
45
+ <dim>-1</dim>
46
+ </port>
47
+ <port id="2" precision="I32">
48
+ <dim>-1</dim>
49
+ </port>
50
+ <port id="3" precision="U8">
51
+ <dim>-1</dim>
52
+ </port>
53
+ </output>
54
+ </layer>
55
+ <layer id="4" name="VocabDecoder_180293" type="VocabDecoder" version="extension">
56
+ <data skip_tokens="100256, 100257, 100258, 100259, 100260, 100261, 100262, 100263, 100264, 100265, 100266, 100267, 100268, 100269, 100270, 100271, 100272, 100273, 100274, 100275, 100276, 100277, 100278, 100279, 100280, 100281, 100282, 100283, 100284, 100285, 100286, 100287, 100288, 100289, 100290, 100291, 100292, 100293, 100294, 100295, 100296, 100297, 100298, 100299, 100300, 100301, 100302, 100303, 100304, 100305, 100306, 100307, 100308, 100309, 100310, 100311, 100312, 100313, 100314, 100315, 100316, 100317, 100318, 100319, 100320, 100321, 100322, 100323, 100324, 100325, 100326, 100327, 100328, 100329, 100330, 100331, 100332, 100333, 100334, 100335, 100336, 100337, 100338, 100339, 100340, 100341, 100342, 100343, 100344, 100345, 100346, 100347, 100348, 100349, 100350, 100351" />
57
+ <input>
58
+ <port id="0" precision="I32">
59
+ <dim>-1</dim>
60
+ <dim>-1</dim>
61
+ </port>
62
+ <port id="1" precision="I32">
63
+ <dim>-1</dim>
64
+ </port>
65
+ <port id="2" precision="I32">
66
+ <dim>-1</dim>
67
+ </port>
68
+ <port id="3" precision="U8">
69
+ <dim>-1</dim>
70
+ </port>
71
+ </input>
72
+ <output>
73
+ <port id="4" precision="I32">
74
+ <dim>-1</dim>
75
+ </port>
76
+ <port id="5" precision="I32">
77
+ <dim>-1</dim>
78
+ </port>
79
+ <port id="6" precision="I32">
80
+ <dim>-1</dim>
81
+ </port>
82
+ <port id="7" precision="I32">
83
+ <dim>-1</dim>
84
+ </port>
85
+ <port id="8" precision="U8">
86
+ <dim>-1</dim>
87
+ </port>
88
+ </output>
89
+ </layer>
90
+ <layer id="5" name="FuzeRagged_180294" type="FuzeRagged" version="extension">
91
+ <input>
92
+ <port id="0" precision="I32">
93
+ <dim>-1</dim>
94
+ </port>
95
+ <port id="1" precision="I32">
96
+ <dim>-1</dim>
97
+ </port>
98
+ <port id="2" precision="I32">
99
+ <dim>-1</dim>
100
+ </port>
101
+ <port id="3" precision="I32">
102
+ <dim>-1</dim>
103
+ </port>
104
+ </input>
105
+ <output>
106
+ <port id="4" precision="I32">
107
+ <dim>-1</dim>
108
+ </port>
109
+ <port id="5" precision="I32">
110
+ <dim>-1</dim>
111
+ </port>
112
+ </output>
113
+ </layer>
114
+ <layer id="6" name="StringTensorPack_180295" type="StringTensorPack" version="extension">
115
+ <data mode="begins_ends" />
116
+ <input>
117
+ <port id="0" precision="I32">
118
+ <dim>-1</dim>
119
+ </port>
120
+ <port id="1" precision="I32">
121
+ <dim>-1</dim>
122
+ </port>
123
+ <port id="2" precision="U8">
124
+ <dim>-1</dim>
125
+ </port>
126
+ </input>
127
+ <output>
128
+ <port id="3" precision="STRING" names="string_output">
129
+ <dim>-1</dim>
130
+ </port>
131
+ </output>
132
+ </layer>
133
+ <layer id="7" name="Result_180296" type="Result" version="opset1">
134
+ <input>
135
+ <port id="0" precision="STRING">
136
+ <dim>-1</dim>
137
+ </port>
138
+ </input>
139
+ </layer>
140
+ </layers>
141
+ <edges>
142
+ <edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
143
+ <edge from-layer="1" from-port="1" to-layer="4" to-port="0" />
144
+ <edge from-layer="2" from-port="0" to-layer="3" to-port="0" />
145
+ <edge from-layer="3" from-port="1" to-layer="4" to-port="1" />
146
+ <edge from-layer="3" from-port="2" to-layer="4" to-port="2" />
147
+ <edge from-layer="3" from-port="3" to-layer="4" to-port="3" />
148
+ <edge from-layer="4" from-port="4" to-layer="5" to-port="0" />
149
+ <edge from-layer="4" from-port="5" to-layer="5" to-port="1" />
150
+ <edge from-layer="4" from-port="6" to-layer="5" to-port="2" />
151
+ <edge from-layer="4" from-port="7" to-layer="5" to-port="3" />
152
+ <edge from-layer="4" from-port="8" to-layer="6" to-port="2" />
153
+ <edge from-layer="5" from-port="4" to-layer="6" to-port="0" />
154
+ <edge from-layer="5" from-port="5" to-layer="6" to-port="1" />
155
+ <edge from-layer="6" from-port="3" to-layer="7" to-port="0" />
156
+ </edges>
157
+ <rt_info>
158
+ <add_attention_mask value="True" />
159
+ <add_prefix_space />
160
+ <add_special_tokens value="True" />
161
+ <bos_token_id value="100257" />
162
+ <chat_template value="{% for message in messages %}{% if (message['role'] == 'system') %}{{'&lt;|im_start|>system&lt;|im_sep|>' + message['content'] + '&lt;|im_end|>'}}{% elif (message['role'] == 'user') %}{{'&lt;|im_start|>user&lt;|im_sep|>' + message['content'] + '&lt;|im_end|>&lt;|im_start|>assistant&lt;|im_sep|>'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '&lt;|im_end|>'}}{% endif %}{% endfor %}" />
163
+ <clean_up_tokenization_spaces />
164
+ <detokenizer_input_type value="i64" />
165
+ <eos_token_id value="100257" />
166
+ <handle_special_tokens_with_re />
167
+ <number_of_inputs value="1" />
168
+ <openvino_tokenizers_version value="2024.5.0.0" />
169
+ <openvino_version value="2024.5.0" />
170
+ <original_tokenizer_class value="&lt;class 'transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast'>" />
171
+ <pad_token_id value="100257" />
172
+ <sentencepiece_version value="0.2.0" />
173
+ <skip_special_tokens value="True" />
174
+ <streaming_detokenizer value="False" />
175
+ <tokenizer_output_type value="i64" />
176
+ <tokenizers_version value="0.19.1" />
177
+ <transformers_version value="4.44.2" />
178
+ <use_max_padding value="False" />
179
+ <use_sentencepiece_backend value="False" />
180
+ <utf8_replace_mode />
181
+ <with_detokenizer value="True" />
182
+ </rt_info>
183
+ </net>
openvino_model.xml ADDED
The diff for this file is too large to render. See raw diff
 
openvino_tokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c6723f10dd17d58c95e8fbb7248b552abe46fa974b9d53ff9555bb37280df0a
3
+ size 2583362
openvino_tokenizer.xml ADDED
@@ -0,0 +1,701 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="tokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_180186" type="Parameter" version="opset1">
5
+ <data shape="?" element_type="string" />
6
+ <output>
7
+ <port id="0" precision="STRING" names="Parameter_180186">
8
+ <dim>-1</dim>
9
+ </port>
10
+ </output>
11
+ </layer>
12
+ <layer id="1" name="Constant_180192" type="Const" version="opset1">
13
+ <data element_type="i32" shape="" offset="0" size="4" />
14
+ <output>
15
+ <port id="0" precision="I32" />
16
+ </output>
17
+ </layer>
18
+ <layer id="2" name="StringTensorUnpack_180187" type="StringTensorUnpack" version="extension">
19
+ <data mode="begins_ends" />
20
+ <input>
21
+ <port id="0" precision="STRING">
22
+ <dim>-1</dim>
23
+ </port>
24
+ </input>
25
+ <output>
26
+ <port id="1" precision="I32">
27
+ <dim>-1</dim>
28
+ </port>
29
+ <port id="2" precision="I32">
30
+ <dim>-1</dim>
31
+ </port>
32
+ <port id="3" precision="U8">
33
+ <dim>-1</dim>
34
+ </port>
35
+ </output>
36
+ </layer>
37
+ <layer id="3" name="ShapeOf_180188" type="ShapeOf" version="opset3">
38
+ <data output_type="i64" />
39
+ <input>
40
+ <port id="0" precision="I32">
41
+ <dim>-1</dim>
42
+ </port>
43
+ </input>
44
+ <output>
45
+ <port id="1" precision="I64">
46
+ <dim>1</dim>
47
+ </port>
48
+ </output>
49
+ </layer>
50
+ <layer id="4" name="Constant_180189" type="Const" version="opset1">
51
+ <data element_type="i32" shape="" offset="0" size="4" />
52
+ <output>
53
+ <port id="0" precision="I32" />
54
+ </output>
55
+ </layer>
56
+ <layer id="5" name="Constant_180190" type="Const" version="opset1">
57
+ <data element_type="i32" shape="" offset="0" size="4" />
58
+ <output>
59
+ <port id="0" precision="I32" />
60
+ </output>
61
+ </layer>
62
+ <layer id="6" name="Gather_180191" type="Gather" version="opset8">
63
+ <data batch_dims="0" />
64
+ <input>
65
+ <port id="0" precision="I64">
66
+ <dim>1</dim>
67
+ </port>
68
+ <port id="1" precision="I32" />
69
+ <port id="2" precision="I32" />
70
+ </input>
71
+ <output>
72
+ <port id="3" precision="I64" />
73
+ </output>
74
+ </layer>
75
+ <layer id="7" name="Constant_180193" type="Const" version="opset1">
76
+ <data element_type="i32" shape="" offset="4" size="4" />
77
+ <output>
78
+ <port id="0" precision="I32" />
79
+ </output>
80
+ </layer>
81
+ <layer id="8" name="Range_180194" type="Range" version="opset4">
82
+ <data output_type="i32" />
83
+ <input>
84
+ <port id="0" precision="I32" />
85
+ <port id="1" precision="I64" />
86
+ <port id="2" precision="I32" />
87
+ </input>
88
+ <output>
89
+ <port id="3" precision="I32">
90
+ <dim>-1</dim>
91
+ </port>
92
+ </output>
93
+ </layer>
94
+ <layer id="9" name="Constant_180195" type="Const" version="opset1">
95
+ <data element_type="i32" shape="" offset="4" size="4" />
96
+ <output>
97
+ <port id="0" precision="I32" />
98
+ </output>
99
+ </layer>
100
+ <layer id="10" name="Constant_180196" type="Const" version="opset1">
101
+ <data element_type="i64" shape="" offset="8" size="8" />
102
+ <output>
103
+ <port id="0" precision="I64" />
104
+ </output>
105
+ </layer>
106
+ <layer id="11" name="Add_180197" type="Add" version="opset1">
107
+ <data auto_broadcast="numpy" />
108
+ <input>
109
+ <port id="0" precision="I64" />
110
+ <port id="1" precision="I64" />
111
+ </input>
112
+ <output>
113
+ <port id="2" precision="I64" />
114
+ </output>
115
+ </layer>
116
+ <layer id="12" name="Constant_180198" type="Const" version="opset1">
117
+ <data element_type="i32" shape="" offset="4" size="4" />
118
+ <output>
119
+ <port id="0" precision="I32" />
120
+ </output>
121
+ </layer>
122
+ <layer id="13" name="Range_180199" type="Range" version="opset4">
123
+ <data output_type="i32" />
124
+ <input>
125
+ <port id="0" precision="I32" />
126
+ <port id="1" precision="I64" />
127
+ <port id="2" precision="I32" />
128
+ </input>
129
+ <output>
130
+ <port id="3" precision="I32">
131
+ <dim>-1</dim>
132
+ </port>
133
+ </output>
134
+ </layer>
135
+ <layer id="14" name="Constant_180261" type="Const" version="opset1">
136
+ <data element_type="u8" shape="3163" offset="16" size="3163" />
137
+ <output>
138
+ <port id="0" precision="U8">
139
+ <dim>3163</dim>
140
+ </port>
141
+ </output>
142
+ </layer>
143
+ <layer id="15" name="SpecialTokensSplit_180262" type="SpecialTokensSplit" version="extension">
144
+ <input>
145
+ <port id="0" precision="I32">
146
+ <dim>-1</dim>
147
+ </port>
148
+ <port id="1" precision="I32">
149
+ <dim>-1</dim>
150
+ </port>
151
+ <port id="2" precision="I32">
152
+ <dim>-1</dim>
153
+ </port>
154
+ <port id="3" precision="I32">
155
+ <dim>-1</dim>
156
+ </port>
157
+ <port id="4" precision="U8">
158
+ <dim>-1</dim>
159
+ </port>
160
+ <port id="5" precision="U8">
161
+ <dim>3163</dim>
162
+ </port>
163
+ </input>
164
+ <output>
165
+ <port id="6" precision="I32">
166
+ <dim>-1</dim>
167
+ </port>
168
+ <port id="7" precision="I32">
169
+ <dim>-1</dim>
170
+ </port>
171
+ <port id="8" precision="I32">
172
+ <dim>-1</dim>
173
+ </port>
174
+ <port id="9" precision="I32">
175
+ <dim>-1</dim>
176
+ </port>
177
+ <port id="10" precision="U8">
178
+ <dim>-1</dim>
179
+ </port>
180
+ <port id="11" precision="BOOL">
181
+ <dim>-1</dim>
182
+ </port>
183
+ </output>
184
+ </layer>
185
+ <layer id="16" name="Constant_180264" type="Const" version="opset1">
186
+ <data element_type="u8" shape="115" offset="3179" size="115" />
187
+ <output>
188
+ <port id="0" precision="U8">
189
+ <dim>115</dim>
190
+ </port>
191
+ </output>
192
+ </layer>
193
+ <layer id="17" name="RegexSplit_180265" type="RegexSplit" version="extension">
194
+ <data behaviour="remove" invert="true" max_splits="-1" />
195
+ <input>
196
+ <port id="0" precision="I32">
197
+ <dim>-1</dim>
198
+ </port>
199
+ <port id="1" precision="I32">
200
+ <dim>-1</dim>
201
+ </port>
202
+ <port id="2" precision="I32">
203
+ <dim>-1</dim>
204
+ </port>
205
+ <port id="3" precision="I32">
206
+ <dim>-1</dim>
207
+ </port>
208
+ <port id="4" precision="U8">
209
+ <dim>-1</dim>
210
+ </port>
211
+ <port id="5" precision="BOOL">
212
+ <dim>-1</dim>
213
+ </port>
214
+ <port id="6" precision="U8">
215
+ <dim>115</dim>
216
+ </port>
217
+ </input>
218
+ <output>
219
+ <port id="7" precision="I32">
220
+ <dim>-1</dim>
221
+ </port>
222
+ <port id="8" precision="I32">
223
+ <dim>-1</dim>
224
+ </port>
225
+ <port id="9" precision="I32">
226
+ <dim>-1</dim>
227
+ </port>
228
+ <port id="10" precision="I32">
229
+ <dim>-1</dim>
230
+ </port>
231
+ <port id="11" precision="U8">
232
+ <dim>-1</dim>
233
+ </port>
234
+ <port id="12" precision="BOOL">
235
+ <dim>-1</dim>
236
+ </port>
237
+ </output>
238
+ </layer>
239
+ <layer id="18" name="Constant_180267" type="Const" version="opset1">
240
+ <data element_type="u8" shape="1046586" offset="3294" size="1046586" />
241
+ <output>
242
+ <port id="0" precision="U8">
243
+ <dim>1046586</dim>
244
+ </port>
245
+ </output>
246
+ </layer>
247
+ <layer id="19" name="StringTensorUnpack_180268" type="StringTensorUnpack" version="extension">
248
+ <data mode="begins_ends" />
249
+ <input>
250
+ <port id="0" precision="U8">
251
+ <dim>1046586</dim>
252
+ </port>
253
+ </input>
254
+ <output>
255
+ <port id="1" precision="I32">
256
+ <dim>-1</dim>
257
+ </port>
258
+ <port id="2" precision="I32">
259
+ <dim>-1</dim>
260
+ </port>
261
+ <port id="3" precision="U8">
262
+ <dim>-1</dim>
263
+ </port>
264
+ </output>
265
+ </layer>
266
+ <layer id="20" name="Constant_180273" type="Const" version="opset1">
267
+ <data element_type="u8" shape="709852" offset="1049880" size="709852" />
268
+ <output>
269
+ <port id="0" precision="U8">
270
+ <dim>709852</dim>
271
+ </port>
272
+ </output>
273
+ </layer>
274
+ <layer id="21" name="StringTensorUnpack_180274" type="StringTensorUnpack" version="extension">
275
+ <data mode="begins_ends" />
276
+ <input>
277
+ <port id="0" precision="U8">
278
+ <dim>709852</dim>
279
+ </port>
280
+ </input>
281
+ <output>
282
+ <port id="1" precision="I32">
283
+ <dim>-1</dim>
284
+ </port>
285
+ <port id="2" precision="I32">
286
+ <dim>-1</dim>
287
+ </port>
288
+ <port id="3" precision="U8">
289
+ <dim>-1</dim>
290
+ </port>
291
+ </output>
292
+ </layer>
293
+ <layer id="22" name="Constant_180276" type="Const" version="opset1">
294
+ <data element_type="u8" shape="733738" offset="1759732" size="733738" />
295
+ <output>
296
+ <port id="0" precision="U8">
297
+ <dim>733738</dim>
298
+ </port>
299
+ </output>
300
+ </layer>
301
+ <layer id="23" name="StringTensorUnpack_180277" type="StringTensorUnpack" version="extension">
302
+ <data mode="begins_ends" />
303
+ <input>
304
+ <port id="0" precision="U8">
305
+ <dim>733738</dim>
306
+ </port>
307
+ </input>
308
+ <output>
309
+ <port id="1" precision="I32">
310
+ <dim>-1</dim>
311
+ </port>
312
+ <port id="2" precision="I32">
313
+ <dim>-1</dim>
314
+ </port>
315
+ <port id="3" precision="U8">
316
+ <dim>-1</dim>
317
+ </port>
318
+ </output>
319
+ </layer>
320
+ <layer id="24" name="Constant_180270" type="Const" version="opset1">
321
+ <data element_type="u8" shape="73372" offset="2493470" size="73372" />
322
+ <output>
323
+ <port id="0" precision="U8">
324
+ <dim>73372</dim>
325
+ </port>
326
+ </output>
327
+ </layer>
328
+ <layer id="25" name="StringTensorUnpack_180271" type="StringTensorUnpack" version="extension">
329
+ <data mode="begins_ends" />
330
+ <input>
331
+ <port id="0" precision="U8">
332
+ <dim>73372</dim>
333
+ </port>
334
+ </input>
335
+ <output>
336
+ <port id="1" precision="I32">
337
+ <dim>-1</dim>
338
+ </port>
339
+ <port id="2" precision="I32">
340
+ <dim>-1</dim>
341
+ </port>
342
+ <port id="3" precision="U8">
343
+ <dim>-1</dim>
344
+ </port>
345
+ </output>
346
+ </layer>
347
+ <layer id="26" name="Constant_180278" type="Const" version="opset1">
348
+ <data element_type="i32" shape="4128" offset="2566842" size="16512" />
349
+ <output>
350
+ <port id="0" precision="I32">
351
+ <dim>4128</dim>
352
+ </port>
353
+ </output>
354
+ </layer>
355
+ <layer id="27" name="BPETokenizer_180279" type="BPETokenizer" version="extension">
356
+ <data unk_token="" fuse_unk="false" suffix_indicator="" end_suffix="" byte_fallback="false" cache_capacity="20070" />
357
+ <input>
358
+ <port id="0" precision="I32">
359
+ <dim>-1</dim>
360
+ </port>
361
+ <port id="1" precision="I32">
362
+ <dim>-1</dim>
363
+ </port>
364
+ <port id="2" precision="I32">
365
+ <dim>-1</dim>
366
+ </port>
367
+ <port id="3" precision="I32">
368
+ <dim>-1</dim>
369
+ </port>
370
+ <port id="4" precision="U8">
371
+ <dim>-1</dim>
372
+ </port>
373
+ <port id="5" precision="I32">
374
+ <dim>-1</dim>
375
+ </port>
376
+ <port id="6" precision="I32">
377
+ <dim>-1</dim>
378
+ </port>
379
+ <port id="7" precision="U8">
380
+ <dim>-1</dim>
381
+ </port>
382
+ <port id="8" precision="I32">
383
+ <dim>-1</dim>
384
+ </port>
385
+ <port id="9" precision="I32">
386
+ <dim>-1</dim>
387
+ </port>
388
+ <port id="10" precision="U8">
389
+ <dim>-1</dim>
390
+ </port>
391
+ <port id="11" precision="I32">
392
+ <dim>-1</dim>
393
+ </port>
394
+ <port id="12" precision="I32">
395
+ <dim>-1</dim>
396
+ </port>
397
+ <port id="13" precision="U8">
398
+ <dim>-1</dim>
399
+ </port>
400
+ <port id="14" precision="I32">
401
+ <dim>-1</dim>
402
+ </port>
403
+ <port id="15" precision="I32">
404
+ <dim>-1</dim>
405
+ </port>
406
+ <port id="16" precision="U8">
407
+ <dim>-1</dim>
408
+ </port>
409
+ <port id="17" precision="I32">
410
+ <dim>4128</dim>
411
+ </port>
412
+ </input>
413
+ <output>
414
+ <port id="18" precision="I32">
415
+ <dim>-1</dim>
416
+ </port>
417
+ <port id="19" precision="I32">
418
+ <dim>-1</dim>
419
+ </port>
420
+ <port id="20" precision="I32">
421
+ <dim>-1</dim>
422
+ </port>
423
+ </output>
424
+ </layer>
425
+ <layer id="28" name="Subtract_180280" type="Subtract" version="opset1">
426
+ <data auto_broadcast="numpy" />
427
+ <input>
428
+ <port id="0" precision="I32">
429
+ <dim>-1</dim>
430
+ </port>
431
+ <port id="1" precision="I32">
432
+ <dim>-1</dim>
433
+ </port>
434
+ </input>
435
+ <output>
436
+ <port id="2" precision="I32">
437
+ <dim>-1</dim>
438
+ </port>
439
+ </output>
440
+ </layer>
441
+ <layer id="29" name="Constant_180281" type="Const" version="opset1">
442
+ <data element_type="i32" shape="" offset="2583354" size="4" />
443
+ <output>
444
+ <port id="0" precision="I32" />
445
+ </output>
446
+ </layer>
447
+ <layer id="30" name="Minimum_180282" type="Minimum" version="opset1">
448
+ <data auto_broadcast="numpy" />
449
+ <input>
450
+ <port id="0" precision="I32">
451
+ <dim>-1</dim>
452
+ </port>
453
+ <port id="1" precision="I32" />
454
+ </input>
455
+ <output>
456
+ <port id="2" precision="I32">
457
+ <dim>-1</dim>
458
+ </port>
459
+ </output>
460
+ </layer>
461
+ <layer id="31" name="Subtract_180283" type="Subtract" version="opset1">
462
+ <data auto_broadcast="numpy" />
463
+ <input>
464
+ <port id="0" precision="I32">
465
+ <dim>-1</dim>
466
+ </port>
467
+ <port id="1" precision="I32">
468
+ <dim>-1</dim>
469
+ </port>
470
+ </input>
471
+ <output>
472
+ <port id="2" precision="I32">
473
+ <dim>-1</dim>
474
+ </port>
475
+ </output>
476
+ </layer>
477
+ <layer id="32" name="Subtract_180284" type="Subtract" version="opset1">
478
+ <data auto_broadcast="numpy" />
479
+ <input>
480
+ <port id="0" precision="I32">
481
+ <dim>-1</dim>
482
+ </port>
483
+ <port id="1" precision="I32">
484
+ <dim>-1</dim>
485
+ </port>
486
+ </input>
487
+ <output>
488
+ <port id="2" precision="I32">
489
+ <dim>-1</dim>
490
+ </port>
491
+ </output>
492
+ </layer>
493
+ <layer id="33" name="Constant_180285" type="Const" version="opset1">
494
+ <data element_type="i32" shape="" offset="0" size="4" />
495
+ <output>
496
+ <port id="0" precision="I32" />
497
+ </output>
498
+ </layer>
499
+ <layer id="34" name="ReduceMax_180286" type="ReduceMax" version="opset1">
500
+ <data keep_dims="false" />
501
+ <input>
502
+ <port id="0" precision="I32">
503
+ <dim>-1</dim>
504
+ </port>
505
+ <port id="1" precision="I32" />
506
+ </input>
507
+ <output>
508
+ <port id="2" precision="I32" />
509
+ </output>
510
+ </layer>
511
+ <layer id="35" name="Constant_180287" type="Const" version="opset1">
512
+ <data element_type="i32" shape="" offset="2583358" size="4" />
513
+ <output>
514
+ <port id="0" precision="I32" />
515
+ </output>
516
+ </layer>
517
+ <layer id="36" name="RaggedToDense_180288" type="RaggedToDense" version="extension">
518
+ <data pad_right="false" />
519
+ <input>
520
+ <port id="0" precision="I32">
521
+ <dim>-1</dim>
522
+ </port>
523
+ <port id="1" precision="I32">
524
+ <dim>-1</dim>
525
+ </port>
526
+ <port id="2" precision="I32">
527
+ <dim>-1</dim>
528
+ </port>
529
+ <port id="3" precision="I32" />
530
+ <port id="4" precision="I32" />
531
+ </input>
532
+ <output>
533
+ <port id="5" precision="I32">
534
+ <dim>-1</dim>
535
+ <dim>-1</dim>
536
+ </port>
537
+ <port id="6" precision="BOOL">
538
+ <dim>-1</dim>
539
+ <dim>-1</dim>
540
+ </port>
541
+ </output>
542
+ </layer>
543
+ <layer id="37" name="Convert_180289" type="Convert" version="opset1">
544
+ <data destination_type="i32" />
545
+ <input>
546
+ <port id="0" precision="BOOL">
547
+ <dim>-1</dim>
548
+ <dim>-1</dim>
549
+ </port>
550
+ </input>
551
+ <output>
552
+ <port id="1" precision="I32">
553
+ <dim>-1</dim>
554
+ <dim>-1</dim>
555
+ </port>
556
+ </output>
557
+ </layer>
558
+ <layer id="38" name="Convert_180289" type="Convert" version="opset1">
559
+ <data destination_type="i64" />
560
+ <input>
561
+ <port id="0" precision="I32">
562
+ <dim>-1</dim>
563
+ <dim>-1</dim>
564
+ </port>
565
+ </input>
566
+ <output>
567
+ <port id="1" precision="I64" names="attention_mask">
568
+ <dim>-1</dim>
569
+ <dim>-1</dim>
570
+ </port>
571
+ </output>
572
+ </layer>
573
+ <layer id="40" name="RaggedToDense_180288.0" type="Convert" version="opset1">
574
+ <data destination_type="i64" />
575
+ <input>
576
+ <port id="0" precision="I32">
577
+ <dim>-1</dim>
578
+ <dim>-1</dim>
579
+ </port>
580
+ </input>
581
+ <output>
582
+ <port id="1" precision="I64" names="input_ids">
583
+ <dim>-1</dim>
584
+ <dim>-1</dim>
585
+ </port>
586
+ </output>
587
+ </layer>
588
+ <layer id="41" name="Result_180290" type="Result" version="opset1">
589
+ <input>
590
+ <port id="0" precision="I64">
591
+ <dim>-1</dim>
592
+ <dim>-1</dim>
593
+ </port>
594
+ </input>
595
+ </layer>
596
+ <layer id="39" name="Result_180291" type="Result" version="opset1">
597
+ <input>
598
+ <port id="0" precision="I64">
599
+ <dim>-1</dim>
600
+ <dim>-1</dim>
601
+ </port>
602
+ </input>
603
+ </layer>
604
+ </layers>
605
+ <edges>
606
+ <edge from-layer="0" from-port="0" to-layer="2" to-port="0" />
607
+ <edge from-layer="1" from-port="0" to-layer="8" to-port="0" />
608
+ <edge from-layer="2" from-port="1" to-layer="3" to-port="0" />
609
+ <edge from-layer="2" from-port="2" to-layer="15" to-port="3" />
610
+ <edge from-layer="2" from-port="1" to-layer="15" to-port="2" />
611
+ <edge from-layer="2" from-port="3" to-layer="15" to-port="4" />
612
+ <edge from-layer="3" from-port="1" to-layer="6" to-port="0" />
613
+ <edge from-layer="4" from-port="0" to-layer="6" to-port="1" />
614
+ <edge from-layer="5" from-port="0" to-layer="6" to-port="2" />
615
+ <edge from-layer="6" from-port="3" to-layer="8" to-port="1" />
616
+ <edge from-layer="6" from-port="3" to-layer="11" to-port="0" />
617
+ <edge from-layer="7" from-port="0" to-layer="8" to-port="2" />
618
+ <edge from-layer="8" from-port="3" to-layer="15" to-port="0" />
619
+ <edge from-layer="9" from-port="0" to-layer="13" to-port="0" />
620
+ <edge from-layer="10" from-port="0" to-layer="11" to-port="1" />
621
+ <edge from-layer="11" from-port="2" to-layer="13" to-port="1" />
622
+ <edge from-layer="12" from-port="0" to-layer="13" to-port="2" />
623
+ <edge from-layer="13" from-port="3" to-layer="15" to-port="1" />
624
+ <edge from-layer="14" from-port="0" to-layer="15" to-port="5" />
625
+ <edge from-layer="15" from-port="6" to-layer="17" to-port="0" />
626
+ <edge from-layer="15" from-port="7" to-layer="17" to-port="1" />
627
+ <edge from-layer="15" from-port="8" to-layer="17" to-port="2" />
628
+ <edge from-layer="15" from-port="9" to-layer="17" to-port="3" />
629
+ <edge from-layer="15" from-port="10" to-layer="17" to-port="4" />
630
+ <edge from-layer="15" from-port="11" to-layer="17" to-port="5" />
631
+ <edge from-layer="16" from-port="0" to-layer="17" to-port="6" />
632
+ <edge from-layer="17" from-port="11" to-layer="27" to-port="4" />
633
+ <edge from-layer="17" from-port="8" to-layer="27" to-port="1" />
634
+ <edge from-layer="17" from-port="9" to-layer="27" to-port="2" />
635
+ <edge from-layer="17" from-port="10" to-layer="27" to-port="3" />
636
+ <edge from-layer="17" from-port="7" to-layer="27" to-port="0" />
637
+ <edge from-layer="18" from-port="0" to-layer="19" to-port="0" />
638
+ <edge from-layer="19" from-port="1" to-layer="27" to-port="5" />
639
+ <edge from-layer="19" from-port="2" to-layer="27" to-port="6" />
640
+ <edge from-layer="19" from-port="3" to-layer="27" to-port="7" />
641
+ <edge from-layer="20" from-port="0" to-layer="21" to-port="0" />
642
+ <edge from-layer="21" from-port="1" to-layer="27" to-port="8" />
643
+ <edge from-layer="21" from-port="2" to-layer="27" to-port="9" />
644
+ <edge from-layer="21" from-port="3" to-layer="27" to-port="10" />
645
+ <edge from-layer="22" from-port="0" to-layer="23" to-port="0" />
646
+ <edge from-layer="23" from-port="1" to-layer="27" to-port="11" />
647
+ <edge from-layer="23" from-port="2" to-layer="27" to-port="12" />
648
+ <edge from-layer="23" from-port="3" to-layer="27" to-port="13" />
649
+ <edge from-layer="24" from-port="0" to-layer="25" to-port="0" />
650
+ <edge from-layer="25" from-port="1" to-layer="27" to-port="14" />
651
+ <edge from-layer="25" from-port="2" to-layer="27" to-port="15" />
652
+ <edge from-layer="25" from-port="3" to-layer="27" to-port="16" />
653
+ <edge from-layer="26" from-port="0" to-layer="27" to-port="17" />
654
+ <edge from-layer="27" from-port="19" to-layer="28" to-port="0" />
655
+ <edge from-layer="27" from-port="18" to-layer="28" to-port="1" />
656
+ <edge from-layer="27" from-port="19" to-layer="36" to-port="1" />
657
+ <edge from-layer="27" from-port="19" to-layer="31" to-port="0" />
658
+ <edge from-layer="27" from-port="19" to-layer="32" to-port="0" />
659
+ <edge from-layer="27" from-port="20" to-layer="36" to-port="2" />
660
+ <edge from-layer="28" from-port="2" to-layer="30" to-port="0" />
661
+ <edge from-layer="29" from-port="0" to-layer="30" to-port="1" />
662
+ <edge from-layer="30" from-port="2" to-layer="31" to-port="1" />
663
+ <edge from-layer="31" from-port="2" to-layer="32" to-port="1" />
664
+ <edge from-layer="31" from-port="2" to-layer="36" to-port="0" />
665
+ <edge from-layer="32" from-port="2" to-layer="34" to-port="0" />
666
+ <edge from-layer="33" from-port="0" to-layer="34" to-port="1" />
667
+ <edge from-layer="34" from-port="2" to-layer="36" to-port="3" />
668
+ <edge from-layer="35" from-port="0" to-layer="36" to-port="4" />
669
+ <edge from-layer="36" from-port="6" to-layer="37" to-port="0" />
670
+ <edge from-layer="36" from-port="5" to-layer="40" to-port="0" />
671
+ <edge from-layer="37" from-port="1" to-layer="38" to-port="0" />
672
+ <edge from-layer="38" from-port="1" to-layer="39" to-port="0" />
673
+ <edge from-layer="40" from-port="1" to-layer="41" to-port="0" />
674
+ </edges>
675
+ <rt_info>
676
+ <add_attention_mask value="True" />
677
+ <add_prefix_space />
678
+ <add_special_tokens value="True" />
679
+ <bos_token_id value="100257" />
680
+ <chat_template value="{% for message in messages %}{% if (message['role'] == 'system') %}{{'&lt;|im_start|>system&lt;|im_sep|>' + message['content'] + '&lt;|im_end|>'}}{% elif (message['role'] == 'user') %}{{'&lt;|im_start|>user&lt;|im_sep|>' + message['content'] + '&lt;|im_end|>&lt;|im_start|>assistant&lt;|im_sep|>'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '&lt;|im_end|>'}}{% endif %}{% endfor %}" />
681
+ <clean_up_tokenization_spaces />
682
+ <detokenizer_input_type value="i64" />
683
+ <eos_token_id value="100257" />
684
+ <handle_special_tokens_with_re />
685
+ <number_of_inputs value="1" />
686
+ <openvino_tokenizers_version value="2024.5.0.0" />
687
+ <openvino_version value="2024.5.0" />
688
+ <original_tokenizer_class value="&lt;class 'transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast'>" />
689
+ <pad_token_id value="100257" />
690
+ <sentencepiece_version value="0.2.0" />
691
+ <skip_special_tokens value="True" />
692
+ <streaming_detokenizer value="False" />
693
+ <tokenizer_output_type value="i64" />
694
+ <tokenizers_version value="0.19.1" />
695
+ <transformers_version value="4.44.2" />
696
+ <use_max_padding value="False" />
697
+ <use_sentencepiece_backend value="False" />
698
+ <utf8_replace_mode />
699
+ <with_detokenizer value="True" />
700
+ </rt_info>
701
+ </net>
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": true,
5
+ "normalized": false,
6
+ "rstrip": true,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": true,
12
+ "normalized": false,
13
+ "rstrip": true,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": true,
19
+ "normalized": false,
20
+ "rstrip": true,
21
+ "single_word": false
22
+ },
23
+ "unk_token": "<|endoftext|>"
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,781 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "100256": {
5
+ "content": "<|dummy_0|>",
6
+ "lstrip": true,
7
+ "normalized": false,
8
+ "rstrip": true,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "100257": {
13
+ "content": "<|endoftext|>",
14
+ "lstrip": true,
15
+ "normalized": false,
16
+ "rstrip": true,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "100258": {
21
+ "content": "<|fim_prefix|>",
22
+ "lstrip": true,
23
+ "normalized": false,
24
+ "rstrip": true,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "100259": {
29
+ "content": "<|fim_middle|>",
30
+ "lstrip": true,
31
+ "normalized": false,
32
+ "rstrip": true,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "100260": {
37
+ "content": "<|fim_suffix|>",
38
+ "lstrip": true,
39
+ "normalized": false,
40
+ "rstrip": true,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "100261": {
45
+ "content": "<|dummy_1|>",
46
+ "lstrip": true,
47
+ "normalized": false,
48
+ "rstrip": true,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "100262": {
53
+ "content": "<|dummy_2|>",
54
+ "lstrip": true,
55
+ "normalized": false,
56
+ "rstrip": true,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "100263": {
61
+ "content": "<|dummy_3|>",
62
+ "lstrip": true,
63
+ "normalized": false,
64
+ "rstrip": true,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "100264": {
69
+ "content": "<|im_start|>",
70
+ "lstrip": true,
71
+ "normalized": false,
72
+ "rstrip": true,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "100265": {
77
+ "content": "<|im_end|>",
78
+ "lstrip": true,
79
+ "normalized": false,
80
+ "rstrip": true,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "100266": {
85
+ "content": "<|im_sep|>",
86
+ "lstrip": true,
87
+ "normalized": false,
88
+ "rstrip": true,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "100267": {
93
+ "content": "<|dummy_4|>",
94
+ "lstrip": true,
95
+ "normalized": false,
96
+ "rstrip": true,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "100268": {
101
+ "content": "<|dummy_5|>",
102
+ "lstrip": true,
103
+ "normalized": false,
104
+ "rstrip": true,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "100269": {
109
+ "content": "<|dummy_6|>",
110
+ "lstrip": true,
111
+ "normalized": false,
112
+ "rstrip": true,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "100270": {
117
+ "content": "<|dummy_7|>",
118
+ "lstrip": true,
119
+ "normalized": false,
120
+ "rstrip": true,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "100271": {
125
+ "content": "<|dummy_8|>",
126
+ "lstrip": true,
127
+ "normalized": false,
128
+ "rstrip": true,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "100272": {
133
+ "content": "<|dummy_9|>",
134
+ "lstrip": true,
135
+ "normalized": false,
136
+ "rstrip": true,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "100273": {
141
+ "content": "<|dummy_10|>",
142
+ "lstrip": true,
143
+ "normalized": false,
144
+ "rstrip": true,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "100274": {
149
+ "content": "<|dummy_11|>",
150
+ "lstrip": true,
151
+ "normalized": false,
152
+ "rstrip": true,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "100275": {
157
+ "content": "<|dummy_12|>",
158
+ "lstrip": true,
159
+ "normalized": false,
160
+ "rstrip": true,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "100276": {
165
+ "content": "<|endofprompt|>",
166
+ "lstrip": true,
167
+ "normalized": false,
168
+ "rstrip": true,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "100277": {
173
+ "content": "<|dummy_13|>",
174
+ "lstrip": true,
175
+ "normalized": false,
176
+ "rstrip": true,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "100278": {
181
+ "content": "<|dummy_14|>",
182
+ "lstrip": true,
183
+ "normalized": false,
184
+ "rstrip": true,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "100279": {
189
+ "content": "<|dummy_15|>",
190
+ "lstrip": true,
191
+ "normalized": false,
192
+ "rstrip": true,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "100280": {
197
+ "content": "<|dummy_16|>",
198
+ "lstrip": true,
199
+ "normalized": false,
200
+ "rstrip": true,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "100281": {
205
+ "content": "<|dummy_17|>",
206
+ "lstrip": true,
207
+ "normalized": false,
208
+ "rstrip": true,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "100282": {
213
+ "content": "<|dummy_18|>",
214
+ "lstrip": true,
215
+ "normalized": false,
216
+ "rstrip": true,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "100283": {
221
+ "content": "<|dummy_19|>",
222
+ "lstrip": true,
223
+ "normalized": false,
224
+ "rstrip": true,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "100284": {
229
+ "content": "<|dummy_20|>",
230
+ "lstrip": true,
231
+ "normalized": false,
232
+ "rstrip": true,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "100285": {
237
+ "content": "<|dummy_21|>",
238
+ "lstrip": true,
239
+ "normalized": false,
240
+ "rstrip": true,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "100286": {
245
+ "content": "<|dummy_22|>",
246
+ "lstrip": true,
247
+ "normalized": false,
248
+ "rstrip": true,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "100287": {
253
+ "content": "<|dummy_23|>",
254
+ "lstrip": true,
255
+ "normalized": false,
256
+ "rstrip": true,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "100288": {
261
+ "content": "<|dummy_24|>",
262
+ "lstrip": true,
263
+ "normalized": false,
264
+ "rstrip": true,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "100289": {
269
+ "content": "<|dummy_25|>",
270
+ "lstrip": true,
271
+ "normalized": false,
272
+ "rstrip": true,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "100290": {
277
+ "content": "<|dummy_26|>",
278
+ "lstrip": true,
279
+ "normalized": false,
280
+ "rstrip": true,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "100291": {
285
+ "content": "<|dummy_27|>",
286
+ "lstrip": true,
287
+ "normalized": false,
288
+ "rstrip": true,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "100292": {
293
+ "content": "<|dummy_28|>",
294
+ "lstrip": true,
295
+ "normalized": false,
296
+ "rstrip": true,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "100293": {
301
+ "content": "<|dummy_29|>",
302
+ "lstrip": true,
303
+ "normalized": false,
304
+ "rstrip": true,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "100294": {
309
+ "content": "<|dummy_30|>",
310
+ "lstrip": true,
311
+ "normalized": false,
312
+ "rstrip": true,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "100295": {
317
+ "content": "<|dummy_31|>",
318
+ "lstrip": true,
319
+ "normalized": false,
320
+ "rstrip": true,
321
+ "single_word": false,
322
+ "special": true
323
+ },
324
+ "100296": {
325
+ "content": "<|dummy_32|>",
326
+ "lstrip": true,
327
+ "normalized": false,
328
+ "rstrip": true,
329
+ "single_word": false,
330
+ "special": true
331
+ },
332
+ "100297": {
333
+ "content": "<|dummy_33|>",
334
+ "lstrip": true,
335
+ "normalized": false,
336
+ "rstrip": true,
337
+ "single_word": false,
338
+ "special": true
339
+ },
340
+ "100298": {
341
+ "content": "<|dummy_34|>",
342
+ "lstrip": true,
343
+ "normalized": false,
344
+ "rstrip": true,
345
+ "single_word": false,
346
+ "special": true
347
+ },
348
+ "100299": {
349
+ "content": "<|dummy_35|>",
350
+ "lstrip": true,
351
+ "normalized": false,
352
+ "rstrip": true,
353
+ "single_word": false,
354
+ "special": true
355
+ },
356
+ "100300": {
357
+ "content": "<|dummy_36|>",
358
+ "lstrip": true,
359
+ "normalized": false,
360
+ "rstrip": true,
361
+ "single_word": false,
362
+ "special": true
363
+ },
364
+ "100301": {
365
+ "content": "<|dummy_37|>",
366
+ "lstrip": true,
367
+ "normalized": false,
368
+ "rstrip": true,
369
+ "single_word": false,
370
+ "special": true
371
+ },
372
+ "100302": {
373
+ "content": "<|dummy_38|>",
374
+ "lstrip": true,
375
+ "normalized": false,
376
+ "rstrip": true,
377
+ "single_word": false,
378
+ "special": true
379
+ },
380
+ "100303": {
381
+ "content": "<|dummy_39|>",
382
+ "lstrip": true,
383
+ "normalized": false,
384
+ "rstrip": true,
385
+ "single_word": false,
386
+ "special": true
387
+ },
388
+ "100304": {
389
+ "content": "<|dummy_40|>",
390
+ "lstrip": true,
391
+ "normalized": false,
392
+ "rstrip": true,
393
+ "single_word": false,
394
+ "special": true
395
+ },
396
+ "100305": {
397
+ "content": "<|dummy_41|>",
398
+ "lstrip": true,
399
+ "normalized": false,
400
+ "rstrip": true,
401
+ "single_word": false,
402
+ "special": true
403
+ },
404
+ "100306": {
405
+ "content": "<|dummy_42|>",
406
+ "lstrip": true,
407
+ "normalized": false,
408
+ "rstrip": true,
409
+ "single_word": false,
410
+ "special": true
411
+ },
412
+ "100307": {
413
+ "content": "<|dummy_43|>",
414
+ "lstrip": true,
415
+ "normalized": false,
416
+ "rstrip": true,
417
+ "single_word": false,
418
+ "special": true
419
+ },
420
+ "100308": {
421
+ "content": "<|dummy_44|>",
422
+ "lstrip": true,
423
+ "normalized": false,
424
+ "rstrip": true,
425
+ "single_word": false,
426
+ "special": true
427
+ },
428
+ "100309": {
429
+ "content": "<|dummy_45|>",
430
+ "lstrip": true,
431
+ "normalized": false,
432
+ "rstrip": true,
433
+ "single_word": false,
434
+ "special": true
435
+ },
436
+ "100310": {
437
+ "content": "<|dummy_46|>",
438
+ "lstrip": true,
439
+ "normalized": false,
440
+ "rstrip": true,
441
+ "single_word": false,
442
+ "special": true
443
+ },
444
+ "100311": {
445
+ "content": "<|dummy_47|>",
446
+ "lstrip": true,
447
+ "normalized": false,
448
+ "rstrip": true,
449
+ "single_word": false,
450
+ "special": true
451
+ },
452
+ "100312": {
453
+ "content": "<|dummy_48|>",
454
+ "lstrip": true,
455
+ "normalized": false,
456
+ "rstrip": true,
457
+ "single_word": false,
458
+ "special": true
459
+ },
460
+ "100313": {
461
+ "content": "<|dummy_49|>",
462
+ "lstrip": true,
463
+ "normalized": false,
464
+ "rstrip": true,
465
+ "single_word": false,
466
+ "special": true
467
+ },
468
+ "100314": {
469
+ "content": "<|dummy_50|>",
470
+ "lstrip": true,
471
+ "normalized": false,
472
+ "rstrip": true,
473
+ "single_word": false,
474
+ "special": true
475
+ },
476
+ "100315": {
477
+ "content": "<|dummy_51|>",
478
+ "lstrip": true,
479
+ "normalized": false,
480
+ "rstrip": true,
481
+ "single_word": false,
482
+ "special": true
483
+ },
484
+ "100316": {
485
+ "content": "<|dummy_52|>",
486
+ "lstrip": true,
487
+ "normalized": false,
488
+ "rstrip": true,
489
+ "single_word": false,
490
+ "special": true
491
+ },
492
+ "100317": {
493
+ "content": "<|dummy_53|>",
494
+ "lstrip": true,
495
+ "normalized": false,
496
+ "rstrip": true,
497
+ "single_word": false,
498
+ "special": true
499
+ },
500
+ "100318": {
501
+ "content": "<|dummy_54|>",
502
+ "lstrip": true,
503
+ "normalized": false,
504
+ "rstrip": true,
505
+ "single_word": false,
506
+ "special": true
507
+ },
508
+ "100319": {
509
+ "content": "<|dummy_55|>",
510
+ "lstrip": true,
511
+ "normalized": false,
512
+ "rstrip": true,
513
+ "single_word": false,
514
+ "special": true
515
+ },
516
+ "100320": {
517
+ "content": "<|dummy_56|>",
518
+ "lstrip": true,
519
+ "normalized": false,
520
+ "rstrip": true,
521
+ "single_word": false,
522
+ "special": true
523
+ },
524
+ "100321": {
525
+ "content": "<|dummy_57|>",
526
+ "lstrip": true,
527
+ "normalized": false,
528
+ "rstrip": true,
529
+ "single_word": false,
530
+ "special": true
531
+ },
532
+ "100322": {
533
+ "content": "<|dummy_58|>",
534
+ "lstrip": true,
535
+ "normalized": false,
536
+ "rstrip": true,
537
+ "single_word": false,
538
+ "special": true
539
+ },
540
+ "100323": {
541
+ "content": "<|dummy_59|>",
542
+ "lstrip": true,
543
+ "normalized": false,
544
+ "rstrip": true,
545
+ "single_word": false,
546
+ "special": true
547
+ },
548
+ "100324": {
549
+ "content": "<|dummy_60|>",
550
+ "lstrip": true,
551
+ "normalized": false,
552
+ "rstrip": true,
553
+ "single_word": false,
554
+ "special": true
555
+ },
556
+ "100325": {
557
+ "content": "<|dummy_61|>",
558
+ "lstrip": true,
559
+ "normalized": false,
560
+ "rstrip": true,
561
+ "single_word": false,
562
+ "special": true
563
+ },
564
+ "100326": {
565
+ "content": "<|dummy_62|>",
566
+ "lstrip": true,
567
+ "normalized": false,
568
+ "rstrip": true,
569
+ "single_word": false,
570
+ "special": true
571
+ },
572
+ "100327": {
573
+ "content": "<|dummy_63|>",
574
+ "lstrip": true,
575
+ "normalized": false,
576
+ "rstrip": true,
577
+ "single_word": false,
578
+ "special": true
579
+ },
580
+ "100328": {
581
+ "content": "<|dummy_64|>",
582
+ "lstrip": true,
583
+ "normalized": false,
584
+ "rstrip": true,
585
+ "single_word": false,
586
+ "special": true
587
+ },
588
+ "100329": {
589
+ "content": "<|dummy_65|>",
590
+ "lstrip": true,
591
+ "normalized": false,
592
+ "rstrip": true,
593
+ "single_word": false,
594
+ "special": true
595
+ },
596
+ "100330": {
597
+ "content": "<|dummy_66|>",
598
+ "lstrip": true,
599
+ "normalized": false,
600
+ "rstrip": true,
601
+ "single_word": false,
602
+ "special": true
603
+ },
604
+ "100331": {
605
+ "content": "<|dummy_67|>",
606
+ "lstrip": true,
607
+ "normalized": false,
608
+ "rstrip": true,
609
+ "single_word": false,
610
+ "special": true
611
+ },
612
+ "100332": {
613
+ "content": "<|dummy_68|>",
614
+ "lstrip": true,
615
+ "normalized": false,
616
+ "rstrip": true,
617
+ "single_word": false,
618
+ "special": true
619
+ },
620
+ "100333": {
621
+ "content": "<|dummy_69|>",
622
+ "lstrip": true,
623
+ "normalized": false,
624
+ "rstrip": true,
625
+ "single_word": false,
626
+ "special": true
627
+ },
628
+ "100334": {
629
+ "content": "<|dummy_70|>",
630
+ "lstrip": true,
631
+ "normalized": false,
632
+ "rstrip": true,
633
+ "single_word": false,
634
+ "special": true
635
+ },
636
+ "100335": {
637
+ "content": "<|dummy_71|>",
638
+ "lstrip": true,
639
+ "normalized": false,
640
+ "rstrip": true,
641
+ "single_word": false,
642
+ "special": true
643
+ },
644
+ "100336": {
645
+ "content": "<|dummy_72|>",
646
+ "lstrip": true,
647
+ "normalized": false,
648
+ "rstrip": true,
649
+ "single_word": false,
650
+ "special": true
651
+ },
652
+ "100337": {
653
+ "content": "<|dummy_73|>",
654
+ "lstrip": true,
655
+ "normalized": false,
656
+ "rstrip": true,
657
+ "single_word": false,
658
+ "special": true
659
+ },
660
+ "100338": {
661
+ "content": "<|dummy_74|>",
662
+ "lstrip": true,
663
+ "normalized": false,
664
+ "rstrip": true,
665
+ "single_word": false,
666
+ "special": true
667
+ },
668
+ "100339": {
669
+ "content": "<|dummy_75|>",
670
+ "lstrip": true,
671
+ "normalized": false,
672
+ "rstrip": true,
673
+ "single_word": false,
674
+ "special": true
675
+ },
676
+ "100340": {
677
+ "content": "<|dummy_76|>",
678
+ "lstrip": true,
679
+ "normalized": false,
680
+ "rstrip": true,
681
+ "single_word": false,
682
+ "special": true
683
+ },
684
+ "100341": {
685
+ "content": "<|dummy_77|>",
686
+ "lstrip": true,
687
+ "normalized": false,
688
+ "rstrip": true,
689
+ "single_word": false,
690
+ "special": true
691
+ },
692
+ "100342": {
693
+ "content": "<|dummy_78|>",
694
+ "lstrip": true,
695
+ "normalized": false,
696
+ "rstrip": true,
697
+ "single_word": false,
698
+ "special": true
699
+ },
700
+ "100343": {
701
+ "content": "<|dummy_79|>",
702
+ "lstrip": true,
703
+ "normalized": false,
704
+ "rstrip": true,
705
+ "single_word": false,
706
+ "special": true
707
+ },
708
+ "100344": {
709
+ "content": "<|dummy_80|>",
710
+ "lstrip": true,
711
+ "normalized": false,
712
+ "rstrip": true,
713
+ "single_word": false,
714
+ "special": true
715
+ },
716
+ "100345": {
717
+ "content": "<|dummy_81|>",
718
+ "lstrip": true,
719
+ "normalized": false,
720
+ "rstrip": true,
721
+ "single_word": false,
722
+ "special": true
723
+ },
724
+ "100346": {
725
+ "content": "<|dummy_82|>",
726
+ "lstrip": true,
727
+ "normalized": false,
728
+ "rstrip": true,
729
+ "single_word": false,
730
+ "special": true
731
+ },
732
+ "100347": {
733
+ "content": "<|dummy_83|>",
734
+ "lstrip": true,
735
+ "normalized": false,
736
+ "rstrip": true,
737
+ "single_word": false,
738
+ "special": true
739
+ },
740
+ "100348": {
741
+ "content": "<|dummy_84|>",
742
+ "lstrip": true,
743
+ "normalized": false,
744
+ "rstrip": true,
745
+ "single_word": false,
746
+ "special": true
747
+ },
748
+ "100349": {
749
+ "content": "<|dummy_85|>",
750
+ "lstrip": true,
751
+ "normalized": false,
752
+ "rstrip": true,
753
+ "single_word": false,
754
+ "special": true
755
+ },
756
+ "100350": {
757
+ "content": "<|dummy_86|>",
758
+ "lstrip": true,
759
+ "normalized": false,
760
+ "rstrip": true,
761
+ "single_word": false,
762
+ "special": true
763
+ },
764
+ "100351": {
765
+ "content": "<|dummy_87|>",
766
+ "lstrip": true,
767
+ "normalized": false,
768
+ "rstrip": true,
769
+ "single_word": false,
770
+ "special": true
771
+ }
772
+ },
773
+ "bos_token": "<|endoftext|>",
774
+ "chat_template": "{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|><|im_start|>assistant<|im_sep|>'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}",
775
+ "clean_up_tokenization_spaces": false,
776
+ "eos_token": "<|endoftext|>",
777
+ "model_max_length": 16384,
778
+ "pad_token": "<|endoftext|>",
779
+ "tokenizer_class": "GPT2Tokenizer",
780
+ "unk_token": "<|endoftext|>"
781
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff