Samuel Cahyawijaya
commited on
Commit
•
7f27ded
1
Parent(s):
525d88c
update indogpt model and tokenizer
Browse files
IndoNLG_finals_indogpt_tokenizer.vocab
DELETED
|
The diff for this file is too large to render.
See raw diff
|
IndoNLG_finals_indogpt_config.json → config.json
RENAMED
|
@@ -27,5 +27,6 @@
|
|
| 27 |
"max_length": 50
|
| 28 |
}
|
| 29 |
},
|
|
|
|
| 30 |
"vocab_size": 40005
|
| 31 |
}
|
| 27 |
"max_length": 50
|
| 28 |
}
|
| 29 |
},
|
| 30 |
+
"tokenizer_class": "IndoNLGTokenizer",
|
| 31 |
"vocab_size": 40005
|
| 32 |
}
|
IndoNLG_finals_indogpt_pytorch_model.bin → pytorch_model.bin
RENAMED
|
File without changes
|
IndoNLG_finals_indogpt_tokenizer.model → sentencepiece.bpe.model
RENAMED
|
File without changes
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}, "additional_special_tokens": ["[java]", "[sunda]", "[indo]", "<mask>"]}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "additional_special_tokens": ["[java]", "[sunda]", "[indo]", "<mask>"]}
|