Update README.md

README.md

@@ -140,6 +140,61 @@ print(tokenizer.decode(outputs[0]))
 
 **Important Note:** Models based on Gemma 2 such as BgGPT-Gemma-2-27B-IT-v1.0 do not support flash attention. Using it results in degraded performance.
 
+# Use with vLLM
+
+Example usage with vLLM:
+
+```python
+from vllm import LLM, SamplingParams
+from vllm.inputs import TokensPrompt
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained(
+    "INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0",
+    use_default_system_prompt=False,
+)
+
+sampling_params = SamplingParams(
+    max_tokens=2048,
+    temperature=0.1,
+    top_k=25,
+    top_p=1,
+    repetition_penalty=1.1,
+    stop_token_ids=[1, 107],
+)
+
+llm = LLM(
+    model="INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0",
+    dtype="bfloat16",
+    enforce_eager=True
+)
+
+messages = [
+    {"role": "user", "content": "Кога е основан Софийският университет?"},
+]
+
+formatted_prompt = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+
+input_ids = tokenizer(
+    formatted_prompt,
+    add_special_tokens=False
+).input_ids
+
+prompt = TokensPrompt(prompt_token_ids=input_ids)
+
+output = llm.generate(
+    prompt,
+    sampling_params
+)
+
+generated_text = output[0].outputs[0].text
+print(generated_text)
+```
+
 # Use with GGML / llama.cpp
 
 The model and instructions for usage in GGUF format are available at [INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0-GGUF](https://huggingface.co/INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0-GGUF).