OwOOwO commited on
Commit
e00a2ae
1 Parent(s): 718524a

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +6 -1
  2. tokenizer_config.json +1 -3
tokenizer.json CHANGED
@@ -1,6 +1,11 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 2048,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
  "padding": null,
10
  "added_tokens": [
11
  {
tokenizer_config.json CHANGED
@@ -1,7 +1,6 @@
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
- "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
  "0": {
7
  "content": "<unk>",
@@ -44,6 +43,5 @@
44
  "truncation_side": "right",
45
  "truncation_strategy": "longest_first",
46
  "unk_token": "<unk>",
47
- "use_default_system_prompt": false,
48
- "use_fast": true
49
  }
 
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
 
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
43
  "truncation_side": "right",
44
  "truncation_strategy": "longest_first",
45
  "unk_token": "<unk>",
46
+ "use_default_system_prompt": false
 
47
  }