Hugging Face's logo

NousResearch
/
Meta-Llama-3-8B-Alternate-Tokenizer

Text Generation
Transformers
Safetensors
PyTorch
English
llama
facebook
meta
llama-3
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
2
teknium commited on
Commit
13a9e8c
•
1 Parent(s): ef35118

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +10 -10
tokenizer_config.json CHANGED
@@ -38,7 +38,7 @@
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
- "special": true
42
  },
43
  "128005": {
44
  "content": "<tool_response>",
@@ -46,7 +46,7 @@
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
- "special": true
50
  },
51
  "128006": {
52
  "content": "<|start_header_id|>",
@@ -70,7 +70,7 @@
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
- "special": true
74
  },
75
  "128009": {
76
  "content": "<|eot_id|>",
@@ -86,7 +86,7 @@
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
- "special": true
90
  },
91
  "128011": {
92
  "content": "</tool_call>",
@@ -94,7 +94,7 @@
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
- "special": true
98
  },
99
  "128012": {
100
  "content": "</tool_response>",
@@ -102,7 +102,7 @@
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
- "special": true
106
  },
107
  "128013": {
108
  "content": "<schema>",
@@ -110,7 +110,7 @@
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
- "special": true
114
  },
115
  "128014": {
116
  "content": "</schema>",
@@ -118,7 +118,7 @@
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
- "special": true
122
  },
123
  "128015": {
124
  "content": "<scratch_pad>",
@@ -126,7 +126,7 @@
126
  "normalized": false,
127
  "rstrip": false,
128
  "single_word": false,
129
- "special": true
130
  },
131
  "128016": {
132
  "content": "</scratch_pad>",
@@ -134,7 +134,7 @@
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
- "special": true
138
  },
139
  "128017": {
140
  "content": "<|reserved_special_token_12|>",
 
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
+ "special": false
42
  },
43
  "128005": {
44
  "content": "<tool_response>",
 
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
+ "special": false
50
  },
51
  "128006": {
52
  "content": "<|start_header_id|>",
 
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
+ "special": false
74
  },
75
  "128009": {
76
  "content": "<|eot_id|>",
 
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
+ "special": false
90
  },
91
  "128011": {
92
  "content": "</tool_call>",
 
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
+ "special": false
98
  },
99
  "128012": {
100
  "content": "</tool_response>",
 
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
+ "special": false
106
  },
107
  "128013": {
108
  "content": "<schema>",
 
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
+ "special": false
114
  },
115
  "128014": {
116
  "content": "</schema>",
 
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
+ "special": false
122
  },
123
  "128015": {
124
  "content": "<scratch_pad>",
 
126
  "normalized": false,
127
  "rstrip": false,
128
  "single_word": false,
129
+ "special": false
130
  },
131
  "128016": {
132
  "content": "</scratch_pad>",
 
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
+ "special": false
138
  },
139
  "128017": {
140
  "content": "<|reserved_special_token_12|>",