Hugging Face's logo

ocisd4
/
mistral_tokenizer_ext_dataprep

Model card Files Community
AlexHT_Hung commited on
Commit
1202640
•
1 Parent(s): 5e7b679

add <|func_start|> <|func_end|>

Browse files
Files changed (4) hide show
  1. added_tokens.json +2 -3
  2. special_tokens_map.json +28 -4
  3. tokenizer.json +19 -1
  4. tokenizer_config.json +16 -0
added_tokens.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "</s>": 2,
3
- "<s>": 1,
4
- "<unk>": 0
5
  }
 
1
  {
2
+ "<|func_end|>": 35712,
3
+ "<|func_start|>": 35713
 
4
  }
special_tokens_map.json CHANGED
@@ -4,8 +4,32 @@
4
  "<s>",
5
  "</s>"
6
  ],
7
- "bos_token": "<s>",
8
- "eos_token": "</s>",
9
- "pad_token": "<unk>",
10
- "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  }
 
4
  "<s>",
5
  "</s>"
6
  ],
7
+ "bos_token": {
8
+ "content": "<s>",
9
+ "lstrip": false,
10
+ "normalized": false,
11
+ "rstrip": false,
12
+ "single_word": false
13
+ },
14
+ "eos_token": {
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "pad_token": {
22
+ "content": "<unk>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "unk_token": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ }
35
  }
tokenizer.json CHANGED
@@ -29,6 +29,24 @@
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
@@ -94803,4 +94821,4 @@
94803
  "▁ ▁▁▁▁▁▁▁▁▁▁"
94804
  ]
94805
  }
94806
- }
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
+ },
33
+ {
34
+ "id": 35712,
35
+ "content": "<|func_end|>",
36
+ "single_word": true,
37
+ "lstrip": true,
38
+ "rstrip": true,
39
+ "normalized": false,
40
+ "special": false
41
+ },
42
+ {
43
+ "id": 35713,
44
+ "content": "<|func_start|>",
45
+ "single_word": true,
46
+ "lstrip": true,
47
+ "rstrip": true,
48
+ "normalized": false,
49
+ "special": false
50
  }
51
  ],
52
  "normalizer": {
 
94821
  "▁ ▁▁▁▁▁▁▁▁▁▁"
94822
  ]
94823
  }
94824
+ }
tokenizer_config.json CHANGED
@@ -25,6 +25,22 @@
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  }
29
  },
30
  "additional_special_tokens": [
 
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
+ },
29
+ "35712": {
30
+ "content": "<|func_end|>",
31
+ "lstrip": true,
32
+ "normalized": false,
33
+ "rstrip": true,
34
+ "single_word": true,
35
+ "special": false
36
+ },
37
+ "35713": {
38
+ "content": "<|func_start|>",
39
+ "lstrip": true,
40
+ "normalized": false,
41
+ "rstrip": true,
42
+ "single_word": true,
43
+ "special": false
44
  }
45
  },
46
  "additional_special_tokens": [