|
## How to use |
|
```python |
|
import transformers |
|
|
|
model = transformers.GPT2LMHeadModel.from_pretrained("lbox/lcube-base") |
|
tokenizer = transformers.AutoTokenizer.from_pretrained( |
|
"lbox/lcube-base", |
|
bos_token="[BOS]", |
|
unk_token="[UNK]", |
|
pad_token="[PAD]", |
|
mask_token="[MASK]", |
|
) |
|
|
|
text = "ํผ๊ณ ์ธ์ ๋ถ์์ง์ ์๋ ์ปคํผ์์์, ํผํด์ B์ผ๋ก๋ถํฐ" |
|
model_inputs = tokenizer(text, |
|
max_length=1024, |
|
padding=True, |
|
truncation=True, |
|
return_tensors='pt') |
|
out = model.generate( |
|
model_inputs["input_ids"], |
|
max_new_tokens=150, |
|
pad_token_id=tokenizer.pad_token_id, |
|
use_cache=True, |
|
repetition_penalty=1.2, |
|
top_k=5, |
|
top_p=0.9, |
|
temperature=1, |
|
num_beams=2, |
|
) |
|
tokenizer.batch_decode(out) |
|
``` |
|
|
|
For more information please visit <https://github.com/lbox-kr/lbox_open>. |
|
|
|
|
|
|
|
## Licensing Information |
|
|
|
Copyright 2022-present LBox Co. Ltd. |
|
|
|
Licensed under the CC BY-NC-ND 4.0 |