teknium commited on
Commit
234dacf
1 Parent(s): 85a591d

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +6 -6
tokenizer.json CHANGED
@@ -46,7 +46,7 @@
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
- "special": true
50
  },
51
  {
52
  "id": 128005,
@@ -55,7 +55,7 @@
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
- "special": true
59
  },
60
  {
61
  "id": 128006,
@@ -82,7 +82,7 @@
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
- "special": true
86
  },
87
  {
88
  "id": 128009,
@@ -100,7 +100,7 @@
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
- "special": true
104
  },
105
  {
106
  "id": 128011,
@@ -109,7 +109,7 @@
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
- "special": true
113
  },
114
  {
115
  "id": 128012,
@@ -118,7 +118,7 @@
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
- "special": true
122
  },
123
  {
124
  "id": 128013,
 
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
+ "special": false
50
  },
51
  {
52
  "id": 128005,
 
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
+ "special": false
59
  },
60
  {
61
  "id": 128006,
 
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
+ "special": false
86
  },
87
  {
88
  "id": 128009,
 
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
+ "special": false
104
  },
105
  {
106
  "id": 128011,
 
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
+ "special": false
113
  },
114
  {
115
  "id": 128012,
 
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
+ "special": false
122
  },
123
  {
124
  "id": 128013,