Hugging Face's logo

NousResearch
/
ByteDance-Seed-OSS-36B-Alternate-Tokenizer

seed_oss
Model card Files
xet
Community
teknium commited on
Commit
d0d0e47
·
verified ·
1 Parent(s): c6b6d4a

Upload 6 files

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +8 -8
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8ff465bd1a8889a3656742c0662e0a29fac05d85978b6bea3c5c91d4a26b4daf
3
- size 11883676
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbcb0d1cf07a32d350059ce7fa434cba1daab5d155f9686041910cf507c19f97
3
+ size 11883663
tokenizer_config.json CHANGED
@@ -78,7 +78,7 @@
78
  "normalized": false,
79
  "rstrip": false,
80
  "single_word": false,
81
- "special": true
82
  },
83
  "10": {
84
  "content": "</tool_response>",
@@ -86,26 +86,26 @@
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
- "special": true
90
  },
91
  "11": {
92
- "content": "<[PLHD11_never_used]>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
- "special": true
98
  },
99
  "12": {
100
- "content": "<[PLHD12_never_used]>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
- "special": true
106
  },
107
  "13": {
108
- "content": "<[PLHD13_never_used]>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
@@ -1027,7 +1027,7 @@
1027
  },
1028
  "bos_token": "<seed:bos>",
1029
  "clean_up_tokenization_spaces": false,
1030
- "eos_token": "<seed:eos>",
1031
  "extra_special_tokens": {},
1032
  "model_max_length": 1000000000000000019884624838656,
1033
  "pad_token": "<seed:pad>",
 
78
  "normalized": false,
79
  "rstrip": false,
80
  "single_word": false,
81
+ "special": false
82
  },
83
  "10": {
84
  "content": "</tool_response>",
 
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
+ "special": false
90
  },
91
  "11": {
92
+ "content": "<|start_header_id|>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
+ "special": false
98
  },
99
  "12": {
100
+ "content": "<|end_header_id|>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
+ "special": false
106
  },
107
  "13": {
108
+ "content": "<|eot_id|>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
 
1027
  },
1028
  "bos_token": "<seed:bos>",
1029
  "clean_up_tokenization_spaces": false,
1030
+ "eos_token": "<|eot_id|>",
1031
  "extra_special_tokens": {},
1032
  "model_max_length": 1000000000000000019884624838656,
1033
  "pad_token": "<seed:pad>",