0x7o commited on
Commit
fdd141f
1 Parent(s): 733437d

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -24
tokenizer_config.json CHANGED
@@ -2063,30 +2063,6 @@
2063
  "rstrip": false,
2064
  "single_word": false,
2065
  "special": true
2066
- },
2067
- "128000": {
2068
- "content": "<|user|>",
2069
- "lstrip": false,
2070
- "normalized": false,
2071
- "rstrip": false,
2072
- "single_word": false,
2073
- "special": true
2074
- },
2075
- "128001": {
2076
- "content": "<|system|>",
2077
- "lstrip": false,
2078
- "normalized": false,
2079
- "rstrip": false,
2080
- "single_word": false,
2081
- "special": true
2082
- },
2083
- "128002": {
2084
- "content": "<|assistant|>",
2085
- "lstrip": false,
2086
- "normalized": false,
2087
- "rstrip": false,
2088
- "single_word": false,
2089
- "special": true
2090
  }
2091
  },
2092
  "additional_special_tokens": [
 
2063
  "rstrip": false,
2064
  "single_word": false,
2065
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2066
  }
2067
  },
2068
  "additional_special_tokens": [