thhaus commited on
Commit
9261b20
1 Parent(s): f61d87e

Upload tokenizer

Browse files
Files changed (3) hide show
  1. .gitattributes +1 -0
  2. tokenizer.json +3 -0
  3. tokenizer_config.json +9 -1
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:656b75fb38a626c862a18a7bac538d650de34dfd5c1211973bff843c9b9211f9
3
+ size 17815145
tokenizer_config.json CHANGED
@@ -3,6 +3,14 @@
3
  "add_eos_token": false,
4
  "add_prefix_space": true,
5
  "added_tokens_decoder": {
 
 
 
 
 
 
 
 
6
  "1": {
7
  "content": "<unk>",
8
  "lstrip": false,
@@ -36,7 +44,7 @@
36
  "pad_token": null,
37
  "sp_model_kwargs": {},
38
  "spaces_between_special_tokens": false,
39
- "tokenizer_class": "NemotronTokenizer",
40
  "unk_token": "<unk>",
41
  "use_default_system_prompt": false
42
  }
 
3
  "add_eos_token": false,
4
  "add_prefix_space": true,
5
  "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<pad>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
  "1": {
15
  "content": "<unk>",
16
  "lstrip": false,
 
44
  "pad_token": null,
45
  "sp_model_kwargs": {},
46
  "spaces_between_special_tokens": false,
47
+ "tokenizer_class": "LlamaTokenizer",
48
  "unk_token": "<unk>",
49
  "use_default_system_prompt": false
50
  }