[](https://github.com/InternLM/xtuner)
## Model
Llama-2-7b-qlora-moss-003-sft is fine-tuned from [Llama-2-7b](https://huggingface.co/meta-llama/Llama-2-7b-hf) with [moss-003-sft](https://huggingface.co/datasets/fnlp/moss-003-sft-data) dataset by [XTuner](https://github.com/InternLM/xtuner).
## Quickstart
### Usage with HuggingFace libraries
```python
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, StoppingCriteria
from transformers.generation import GenerationConfig
class StopWordStoppingCriteria(StoppingCriteria):
def __init__(self, tokenizer, stop_word):
self.tokenizer = tokenizer
self.stop_word = stop_word
self.length = len(self.stop_word)
def __call__(self, input_ids, *args, **kwargs) -> bool:
cur_text = self.tokenizer.decode(input_ids[0])
cur_text = cur_text.replace('\r', '').replace('\n', '')
return cur_text[-self.length:] == self.stop_word
tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-2-7b-hf', trust_remote_code=True)
quantization_config = BitsAndBytesConfig(load_in_4bit=True, load_in_8bit=False, llm_int8_threshold=6.0, llm_int8_has_fp16_weight=False, bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type='nf4')
model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-2-7b-hf', quantization_config=quantization_config, device_map='auto', trust_remote_code=True).eval()
model = PeftModel.from_pretrained(model, 'xtuner/Llama-2-7b-qlora-moss-003-sft')
gen_config = GenerationConfig(max_new_tokens=1024, do_sample=True, temperature=0.1, top_p=0.75, top_k=40)
# Note: In this example, we disable the use of plugins because the API depends on additional implementations.
# If you want to experience plugins, please refer to XTuner CLI!
prompt_template = (
'You are an AI assistant whose name is Llama2.\n'
'Capabilities and tools that Llama2 can possess.\n'
'- Inner thoughts: disabled.\n'
'- Web search: disabled.\n'
'- Calculator: disabled.\n'
'- Equation solver: disabled.\n'
'- Text-to-image: disabled.\n'
'- Image edition: disabled.\n'
'- Text-to-speech: disabled.\n'
'<|Human|>: {input}