malhajar commited on
Commit
32cdb5f
1 Parent(s): d22ebae

basic training

Browse files
special_tokens_map.json CHANGED
@@ -13,7 +13,13 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
- "pad_token": "</s>",
 
 
 
 
 
 
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
  "unk_token": {
24
  "content": "<unk>",
25
  "lstrip": false,
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -34,11 +34,11 @@
34
  "legacy": true,
35
  "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": "</s>",
 
37
  "sp_model_kwargs": {},
38
  "spaces_between_special_tokens": false,
 
39
  "tokenizer_class": "LlamaTokenizer",
40
- "trust_remote_code": true,
41
  "unk_token": "<unk>",
42
- "use_default_system_prompt": false,
43
- "use_fast": true
44
  }
 
34
  "legacy": true,
35
  "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": "</s>",
37
+ "padding_side": "right",
38
  "sp_model_kwargs": {},
39
  "spaces_between_special_tokens": false,
40
+ "split_special_tokens": false,
41
  "tokenizer_class": "LlamaTokenizer",
 
42
  "unk_token": "<unk>",
43
+ "use_default_system_prompt": false
 
44
  }