venkycs commited on
Commit
6177445
·
verified ·
1 Parent(s): 31bb118

Upload 3 files

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +21 -0
  2. tokenizer.json +2 -2
  3. tokenizer_config.json +36 -9
special_tokens_map.json CHANGED
@@ -1,5 +1,26 @@
1
  {
2
  "additional_special_tokens": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  {
4
  "content": "<Impersonate>",
5
  "lstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
+ {
4
+ "content": "<|start_header_id|>system<|end_header_id|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "<|start_header_id|>user<|end_header_id|>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ },
17
+ {
18
+ "content": "<|start_header_id|>assistant<|end_header_id|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
  {
25
  "content": "<Impersonate>",
26
  "lstrip": false,
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ebc33ed74d47a16e4d7d0a35f93890e99857cfd180f665622c8bfd90f6395deb
3
- size 17211910
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d22c15bc3b7a990838e5ad5fa1123784d75fff1e1668446346a1f8d26953df0
3
+ size 17212469
tokenizer_config.json CHANGED
@@ -2050,7 +2050,7 @@
2050
  "special": true
2051
  },
2052
  "128256": {
2053
- "content": "<Impersonate>",
2054
  "lstrip": false,
2055
  "normalized": false,
2056
  "rstrip": false,
@@ -2058,7 +2058,7 @@
2058
  "special": true
2059
  },
2060
  "128257": {
2061
- "content": "</Impersonate>",
2062
  "lstrip": false,
2063
  "normalized": false,
2064
  "rstrip": false,
@@ -2066,7 +2066,7 @@
2066
  "special": true
2067
  },
2068
  "128258": {
2069
- "content": "<Knowledge>",
2070
  "lstrip": false,
2071
  "normalized": false,
2072
  "rstrip": false,
@@ -2074,7 +2074,7 @@
2074
  "special": true
2075
  },
2076
  "128259": {
2077
- "content": "</Knowledge>",
2078
  "lstrip": false,
2079
  "normalized": false,
2080
  "rstrip": false,
@@ -2082,7 +2082,7 @@
2082
  "special": true
2083
  },
2084
  "128260": {
2085
- "content": "<Intent>",
2086
  "lstrip": false,
2087
  "normalized": false,
2088
  "rstrip": false,
@@ -2090,7 +2090,7 @@
2090
  "special": true
2091
  },
2092
  "128261": {
2093
- "content": "</Intent>",
2094
  "lstrip": false,
2095
  "normalized": false,
2096
  "rstrip": false,
@@ -2098,7 +2098,7 @@
2098
  "special": true
2099
  },
2100
  "128262": {
2101
- "content": "<Reasoning>",
2102
  "lstrip": false,
2103
  "normalized": false,
2104
  "rstrip": false,
@@ -2106,7 +2106,7 @@
2106
  "special": true
2107
  },
2108
  "128263": {
2109
- "content": "</Reasoning>",
2110
  "lstrip": false,
2111
  "normalized": false,
2112
  "rstrip": false,
@@ -2114,7 +2114,7 @@
2114
  "special": true
2115
  },
2116
  "128264": {
2117
- "content": "<AvoidMistakes>",
2118
  "lstrip": false,
2119
  "normalized": false,
2120
  "rstrip": false,
@@ -2122,6 +2122,30 @@
2122
  "special": true
2123
  },
2124
  "128265": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2125
  "content": "</AvoidMistakes>",
2126
  "lstrip": false,
2127
  "normalized": false,
@@ -2131,6 +2155,9 @@
2131
  }
2132
  },
2133
  "additional_special_tokens": [
 
 
 
2134
  "<Impersonate>",
2135
  "</Impersonate>",
2136
  "<Knowledge>",
 
2050
  "special": true
2051
  },
2052
  "128256": {
2053
+ "content": "<|start_header_id|>system<|end_header_id|>",
2054
  "lstrip": false,
2055
  "normalized": false,
2056
  "rstrip": false,
 
2058
  "special": true
2059
  },
2060
  "128257": {
2061
+ "content": "<|start_header_id|>user<|end_header_id|>",
2062
  "lstrip": false,
2063
  "normalized": false,
2064
  "rstrip": false,
 
2066
  "special": true
2067
  },
2068
  "128258": {
2069
+ "content": "<|start_header_id|>assistant<|end_header_id|>",
2070
  "lstrip": false,
2071
  "normalized": false,
2072
  "rstrip": false,
 
2074
  "special": true
2075
  },
2076
  "128259": {
2077
+ "content": "<Impersonate>",
2078
  "lstrip": false,
2079
  "normalized": false,
2080
  "rstrip": false,
 
2082
  "special": true
2083
  },
2084
  "128260": {
2085
+ "content": "</Impersonate>",
2086
  "lstrip": false,
2087
  "normalized": false,
2088
  "rstrip": false,
 
2090
  "special": true
2091
  },
2092
  "128261": {
2093
+ "content": "<Knowledge>",
2094
  "lstrip": false,
2095
  "normalized": false,
2096
  "rstrip": false,
 
2098
  "special": true
2099
  },
2100
  "128262": {
2101
+ "content": "</Knowledge>",
2102
  "lstrip": false,
2103
  "normalized": false,
2104
  "rstrip": false,
 
2106
  "special": true
2107
  },
2108
  "128263": {
2109
+ "content": "<Intent>",
2110
  "lstrip": false,
2111
  "normalized": false,
2112
  "rstrip": false,
 
2114
  "special": true
2115
  },
2116
  "128264": {
2117
+ "content": "</Intent>",
2118
  "lstrip": false,
2119
  "normalized": false,
2120
  "rstrip": false,
 
2122
  "special": true
2123
  },
2124
  "128265": {
2125
+ "content": "<Reasoning>",
2126
+ "lstrip": false,
2127
+ "normalized": false,
2128
+ "rstrip": false,
2129
+ "single_word": false,
2130
+ "special": true
2131
+ },
2132
+ "128266": {
2133
+ "content": "</Reasoning>",
2134
+ "lstrip": false,
2135
+ "normalized": false,
2136
+ "rstrip": false,
2137
+ "single_word": false,
2138
+ "special": true
2139
+ },
2140
+ "128267": {
2141
+ "content": "<AvoidMistakes>",
2142
+ "lstrip": false,
2143
+ "normalized": false,
2144
+ "rstrip": false,
2145
+ "single_word": false,
2146
+ "special": true
2147
+ },
2148
+ "128268": {
2149
  "content": "</AvoidMistakes>",
2150
  "lstrip": false,
2151
  "normalized": false,
 
2155
  }
2156
  },
2157
  "additional_special_tokens": [
2158
+ "<|start_header_id|>system<|end_header_id|>",
2159
+ "<|start_header_id|>user<|end_header_id|>",
2160
+ "<|start_header_id|>assistant<|end_header_id|>",
2161
  "<Impersonate>",
2162
  "</Impersonate>",
2163
  "<Knowledge>",