c0d3r69 commited on
Commit
9d8123c
1 Parent(s): eedfda2

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -1
  2. tokenizer_config.json +2 -3
tokenizer.json CHANGED
@@ -46,7 +46,7 @@
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
- "special": true
50
  }
51
  ],
52
  "normalizer": {
 
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
+ "special": false
50
  }
51
  ],
52
  "normalizer": {
tokenizer_config.json CHANGED
@@ -41,7 +41,7 @@
41
  "normalized": false,
42
  "rstrip": false,
43
  "single_word": false,
44
- "special": true
45
  }
46
  },
47
  "additional_special_tokens": [],
@@ -55,8 +55,7 @@
55
  "sp_model_kwargs": {},
56
  "spaces_between_special_tokens": false,
57
  "tokenizer_class": "LlamaTokenizer",
58
- "trust_remote_code": false,
59
  "unk_token": "<unk>",
60
- "use_default_system_prompt": true,
61
  "use_fast": true
62
  }
 
41
  "normalized": false,
42
  "rstrip": false,
43
  "single_word": false,
44
+ "special": false
45
  }
46
  },
47
  "additional_special_tokens": [],
 
55
  "sp_model_kwargs": {},
56
  "spaces_between_special_tokens": false,
57
  "tokenizer_class": "LlamaTokenizer",
 
58
  "unk_token": "<unk>",
59
+ "use_default_system_prompt": false,
60
  "use_fast": true
61
  }