meta-llama

/

Meta-Llama-3-8B

like 4.37k

I following the code provided by example:

import transformers
import torch

base_model = "meta-llama/Meta-Llama-3-8B"
pipeline = transformers.pipeline("text-generation", model=base_model, 
                                 model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto")
pipeline("Hey how are you doing today?")

But there is error:

Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.
Unexpected exception formatting exception. Falling back to standard exception
Traceback (most recent call last):
  File "/opt/conda/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3526, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/var/tmp/ipykernel_28044/1028330205.py", line 1, in <module>
    pipeline("Hey how are you doing today?")
  File "/opt/conda/lib/python3.10/site-packages/transformers/pipelines/text_generation.py", line 241, in __call__
  File "/opt/conda/lib/python3.10/site-packages/transformers/pipelines/base.py", line 1196, in __call__
  File "/opt/conda/lib/python3.10/site-packages/transformers/pipelines/base.py", line 1203, in run_single
    if self.call_count > 10 and self.framework == "pt" and self.device.type == "cuda":
  File "/opt/conda/lib/python3.10/site-packages/transformers/pipelines/base.py", line 1102, in forward
    It is not meant to be called directly, it will be automatically called and the final parameters resolved by
  File "/opt/conda/lib/python3.10/site-packages/transformers/pipelines/text_generation.py", line 328, in _forward
    out_b = generated_sequence.shape[0]
  File "/opt/conda/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py", line 1597, in generate
    logits_processor=prepared_logits_processor,
  File "/opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py", line 2715, in sample
    ['Today is a beautiful day, and we must do everything possible to make it a day of celebration.']
  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py", line 166, in new_forward
    output = module._old_forward(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1208, in forward
    outputs = self.model(
  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 992, in forward
    causal_mask = self._update_causal_mask(attention_mask, inputs_embeds, cache_position, past_seen_tokens)
  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 1076, in _update_causal_mask
    if AttentionMaskConverter._ignore_causal_mask_sdpa(
AttributeError: type object 'AttentionMaskConverter' has no attribute '_ignore_causal_mask_sdpa'