Gnider commited on
Commit
2db3eed
1 Parent(s): 8c6f7d1

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +9 -0
tokenizer_config.json CHANGED
@@ -41,6 +41,14 @@
41
  "rstrip": false,
42
  "single_word": false,
43
  "special": true
 
 
 
 
 
 
 
 
44
  }
45
  },
46
  "bos_token": "<s>",
@@ -55,4 +63,5 @@
55
  "truncation_side": "left",
56
  "trust_remote_code": false,
57
  "unk_token": "<unk>"
 
58
  }
 
41
  "rstrip": false,
42
  "single_word": false,
43
  "special": true
44
+ },
45
+ "5": {
46
+ "content": "<sep>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
  }
53
  },
54
  "bos_token": "<s>",
 
63
  "truncation_side": "left",
64
  "trust_remote_code": false,
65
  "unk_token": "<unk>"
66
+ "sep_token": "<sep>"
67
  }