togethercomputer

/

StripedHyena-Nous-7B

like 135

%%time
!pip install git+https://github.com/huggingface/transformers -qqq
# trl
!pip install git+https://github.com/huggingface/trl -qqq
!pip install datasets peft accelerate safetensors --upgrade -qqq
!pip install ninja packaging --upgrade -qqq
!pip install sentencepiece bitsandbytes -qqq
!pip install -U xformers deepspeed -qqq
!pip install attention_sinks -qqq
!python -c "import torch; assert torch.cuda.get_device_capability()[0] >= 8, 'Hardware not supported for Flash Attention'"

!export CUDA_HOME=/usr/local/cuda-11.8
!MAX_JOBS=4 pip install flash-attn --no-build-isolation -qqq
# !pip install git+"https://github.com/Dao-AILab/flash-attention.git"

load model

from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    HfArgumentParser,
    TrainingArguments,
    pipeline,
    logging,
    GenerationConfig,
    TextIteratorStreamer,
)
# from attention_sinks import AutoModelForCausalLM

import torch

model_ID_1="togethercomputer/StripedHyena-Nous-7B"

model = AutoModelForCausalLM.from_pretrained(model_ID_1,
                                             device_map="auto",
                                             trust_remote_code=True,
                                             torch_dtype=torch.bfloat16,
                                             use_flash_attention_2=True, # True , False
                                             low_cpu_mem_usage= True,
                                             )



max_length=2048 #get_max_length()
print("max_length",max_length)


tokenizer = AutoTokenizer.from_pretrained(model_ID_1,
                                          # use_fast = False, # True False
                                          max_length=max_length,)

Error: