mistralai

/

Mixtral-8x7B-Instruct-v0.1

like 3.9k

python test_mixtral.py
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "/data1/apps/huggingface/Mixtral-8x7B-Instruct-v0.1"
tokenizer = AutoTokenizer.from_pretrained(model_id)

model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True)

text = "Hello my name is"
inputs = tokenizer(text, return_tensors="pt").to(0)

outputs = model.generate(**inputs, max_new_tokens=20)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))

Loading checkpoint shards: 79%|███████████████████████████████████████████████████████████████████████████ | 15/19 [00:33<00:08, 2.24s/it]
Traceback (most recent call last):
File "/home/ubuntu/test_mixtral.py", line 7, in
model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True)
File "/data1/apps/miniconda3/envs/Mixtral/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 566, in from_pretrained
return model_class.from_pretrained(
File "/data1/apps/miniconda3/envs/Mixtral/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3694, in from_pretrained
) = cls._load_pretrained_model(
File "/data1/apps/miniconda3/envs/Mixtral/lib/python3.10/site-packages/transformers/modeling_utils.py", line 4079, in _load_pretrained_model
state_dict = load_state_dict(shard_file)
File "/data1/apps/miniconda3/envs/Mixtral/lib/python3.10/site-packages/transformers/modeling_utils.py", line 503, in load_state_dict
with safe_open(checkpoint_file, framework="pt") as f:
safetensors_rust.SafetensorError: Error while deserializing header: MetadataIncompleteBuffer

Can anyone know why?