Hugging Face's logo

MSey
/
tiny_BROLLLT_0001.1

Text Generation
Transformers
Safetensors
ger
llama
Inference Endpoints
text-generation-inference
Model card Files Community
MSey commited on
Commit
8b076de
•
1 Parent(s): 980665c

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +54 -54
tokenizer_config.json CHANGED
@@ -1,54 +1,54 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "added_tokens_decoder": {
5
- "0": {
6
- "content": "<unk>",
7
- "lstrip": false,
8
- "normalized": false,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "1": {
14
- "content": "<s>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "2": {
22
- "content": "</s>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "32000": {
30
- "content": "[PAD]",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- }
37
- },
38
- "bos_token": "<s>",
39
- "chat_template": "{{bos_token}}{{'### Content\n'}}{{messages}}{{'\n\n### Answer'}}",
40
- "clean_up_tokenization_spaces": false,
41
- "eos_token": "</s>",
42
- "legacy": false,
43
- "max_length": 591,
44
- "model_max_length": 1000000000000000019884624838656,
45
- "pad_token": "[PAD]",
46
- "padding_side": "left",
47
- "sp_model_kwargs": {},
48
- "stride": 0,
49
- "tokenizer_class": "LlamaTokenizer",
50
- "truncation_side": "right",
51
- "truncation_strategy": "longest_first",
52
- "unk_token": "<unk>",
53
- "use_default_system_prompt": false
54
- }
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "[PAD]",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ }
37
+ },
38
+ "bos_token": "<s>",
39
+ "chat_template": None,
40
+ "clean_up_tokenization_spaces": false,
41
+ "eos_token": "</s>",
42
+ "legacy": false,
43
+ "max_length": 591,
44
+ "model_max_length": 1000000000000000019884624838656,
45
+ "pad_token": "[PAD]",
46
+ "padding_side": "left",
47
+ "sp_model_kwargs": {},
48
+ "stride": 0,
49
+ "tokenizer_class": "LlamaTokenizer",
50
+ "truncation_side": "right",
51
+ "truncation_strategy": "longest_first",
52
+ "unk_token": "<unk>",
53
+ "use_default_system_prompt": false
54
+ }