Upload tokenizer.json
Browse files- tokenizer.json +6 -6
tokenizer.json
CHANGED
@@ -46,7 +46,7 @@
|
|
46 |
"lstrip": false,
|
47 |
"rstrip": false,
|
48 |
"normalized": false,
|
49 |
-
"special":
|
50 |
},
|
51 |
{
|
52 |
"id": 128005,
|
@@ -55,7 +55,7 @@
|
|
55 |
"lstrip": false,
|
56 |
"rstrip": false,
|
57 |
"normalized": false,
|
58 |
-
"special":
|
59 |
},
|
60 |
{
|
61 |
"id": 128006,
|
@@ -82,7 +82,7 @@
|
|
82 |
"lstrip": false,
|
83 |
"rstrip": false,
|
84 |
"normalized": false,
|
85 |
-
"special":
|
86 |
},
|
87 |
{
|
88 |
"id": 128009,
|
@@ -100,7 +100,7 @@
|
|
100 |
"lstrip": false,
|
101 |
"rstrip": false,
|
102 |
"normalized": false,
|
103 |
-
"special":
|
104 |
},
|
105 |
{
|
106 |
"id": 128011,
|
@@ -109,7 +109,7 @@
|
|
109 |
"lstrip": false,
|
110 |
"rstrip": false,
|
111 |
"normalized": false,
|
112 |
-
"special":
|
113 |
},
|
114 |
{
|
115 |
"id": 128012,
|
@@ -118,7 +118,7 @@
|
|
118 |
"lstrip": false,
|
119 |
"rstrip": false,
|
120 |
"normalized": false,
|
121 |
-
"special":
|
122 |
},
|
123 |
{
|
124 |
"id": 128013,
|
|
|
46 |
"lstrip": false,
|
47 |
"rstrip": false,
|
48 |
"normalized": false,
|
49 |
+
"special": false
|
50 |
},
|
51 |
{
|
52 |
"id": 128005,
|
|
|
55 |
"lstrip": false,
|
56 |
"rstrip": false,
|
57 |
"normalized": false,
|
58 |
+
"special": false
|
59 |
},
|
60 |
{
|
61 |
"id": 128006,
|
|
|
82 |
"lstrip": false,
|
83 |
"rstrip": false,
|
84 |
"normalized": false,
|
85 |
+
"special": false
|
86 |
},
|
87 |
{
|
88 |
"id": 128009,
|
|
|
100 |
"lstrip": false,
|
101 |
"rstrip": false,
|
102 |
"normalized": false,
|
103 |
+
"special": false
|
104 |
},
|
105 |
{
|
106 |
"id": 128011,
|
|
|
109 |
"lstrip": false,
|
110 |
"rstrip": false,
|
111 |
"normalized": false,
|
112 |
+
"special": false
|
113 |
},
|
114 |
{
|
115 |
"id": 128012,
|
|
|
118 |
"lstrip": false,
|
119 |
"rstrip": false,
|
120 |
"normalized": false,
|
121 |
+
"special": false
|
122 |
},
|
123 |
{
|
124 |
"id": 128013,
|