vllm_test.py

from openai import OpenAI

# test file in cas I need to run entirely locally
# this test comptaibility with open ai

client = OpenAI(
    base_url="http://192.168.1.39:18000/v1",
    api_key="token-abc123",
)

completion = client.chat.completions.create(
  model="Qwen/Qwen2.5-1.5B-Instruct",
  messages=[
    {"role": "user", "content": "Hello!"}
  ]
)

print(completion.choices[0].message)