Xenova HF staff commited on
Commit
2b6ac3f
1 Parent(s): 17a0165

Set `<start_of_turn` and `<end_of_turn>` to be special tokens (#2)

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +2 -2
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2f1f411054535bfe0e360e3431e7340cfc795d0d4ed1ebd426b01993831d49e4
3
- size 17525359
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cf8cddb82b68ab495546855952d707acfa17f23c97031e0cf346b207d73faae
3
+ size 17525357
tokenizer_config.json CHANGED
@@ -856,7 +856,7 @@
856
  "normalized": false,
857
  "rstrip": false,
858
  "single_word": false,
859
- "special": false
860
  },
861
  "107": {
862
  "content": "<end_of_turn>",
@@ -864,7 +864,7 @@
864
  "normalized": false,
865
  "rstrip": false,
866
  "single_word": false,
867
- "special": false
868
  },
869
  "108": {
870
  "content": "\n",
 
856
  "normalized": false,
857
  "rstrip": false,
858
  "single_word": false,
859
+ "special": true
860
  },
861
  "107": {
862
  "content": "<end_of_turn>",
 
864
  "normalized": false,
865
  "rstrip": false,
866
  "single_word": false,
867
+ "special": true
868
  },
869
  "108": {
870
  "content": "\n",