croissantllm

/

CroissantLLMChat-v0.1

like 45

Hi,
I try to use the model as translation agent and I define a system prompt for the instruction, but it didn't works.
What is the good way to define instruct with this model ?
Thanks

Script

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer


model_name = "croissantllm/CroissantLLMChat-v0.1"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)


generation_args = {
    "max_new_tokens": 256,
    "do_sample": True,
    "temperature": 0.3,
    "top_p": 0.90,
    "top_k": 40,
    "repetition_penalty": 1.05,
    "eos_token_id": [tokenizer.eos_token_id, 32000],
}

chat = [
   {"role": "system", "content": "Translates next content into English"},
   {"role": "user", "content": "Une pomme verte"},
]

chat_input = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)

inputs = tokenizer(chat_input, return_tensors="pt").to(model.device)
tokens = model.generate(**inputs, **generation_args)

print(tokenizer.decode(tokens[0]))
# print tokens individually
print([(tokenizer.decode([tok]), tok) for tok in tokens[0].tolist()])

Output

Loading checkpoint shards: 100%|██████████████████| 2/2 [00:00<00:00,  4.07it/s]
Setting `pad_token_id` to `eos_token_id`:32000 for open-end generation.
<s><|im_start|> system
Translates next content into English<|im_end|> 
<|im_start|> user
Une pomme verte<|im_end|> 
<|im_start|> assistant
 Une orange.<|im_end|>
[('<s>', 1), ('<|im_start|>', 32001), ('system', 2935), ('\nTrans', 19814), ('l', 183), ('ates', 2026), ('next', 3314), ('content', 4097), ('into', 2127), ('English', 6938), ('<|im_end|>', 32000), ('\n', 1424), ('<|im_start|>', 32001), ('user', 3592), ('\nUne', 7488), ('p', 1113), ('omme', 4889), ('verte', 23140), ('<|im_end|>', 32000), ('\n', 1424), ('<|im_start|>', 32001), ('assistant', 20036), ('\n', 106), ('Une', 3559), ('orange', 18401), ('.', 121), ('<|im_end|>', 32000)]