EleutherAI
/

polyglot-ko-1.3b

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

hyunwoongko commited on Sep 15, 2022

Commit

7860e50

•

1 Parent(s): 6a727c1

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +2 -2

tokenizer_config.json CHANGED Viewed

@@ -1,7 +1,7 @@
 {
-  "eos_token": "<eos>",
   "name_or_path": "EleutherAI/gpt-neox-ko-1.3b",
-  "pad_token": "<pad>",
   "special_tokens_map_file": "/fsx/kevinai/.cache/models--EleutherAI--gpt-neox-ko-1.3b/snapshots/1736ae3c92f641c994d1361fb5d2c680fd483ab0/special_tokens_map.json",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

 {
+  "eos_token": "<|endoftext|>",
   "name_or_path": "EleutherAI/gpt-neox-ko-1.3b",
+  "pad_token": "<|endoftext|>>",
   "special_tokens_map_file": "/fsx/kevinai/.cache/models--EleutherAI--gpt-neox-ko-1.3b/snapshots/1736ae3c92f641c994d1361fb5d2c680fd483ab0/special_tokens_map.json",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }