TrishanuDas/tayavision-multilingual-merged-0.7

TayaVision - Tiny Aya Vision (Multilingual Merged)

Merged multilingual VLM with α=0.7 (70% fine-tuned, 30% Aya Global).

LLM backbone weights are linearly interpolated between the multilingual fine-tuned model and CohereLabs/tiny-aya-global. Vision encoder and connector weights are kept from the fine-tuned model.

import torch
from PIL import Image
from transformers import AutoModelForCausalLM, AutoProcessor

repo = "TrishanuDas/tayavision-multilingual-merged"

model = AutoModelForCausalLM.from_pretrained(repo, torch_dtype=torch.bfloat16, trust_remote_code=True)
model = model.to("cuda").eval()

processor = AutoProcessor.from_pretrained(repo, trust_remote_code=True)

image = Image.open("your_image.jpg").convert("RGB")

messages = [
    {"role": "user", "content": [
        {"type": "image"},
        {"type": "text", "text": "Describe this image in detail."},
    ]},
]

inputs = processor.apply_chat_template(
    messages,
    images=image,
    add_generation_prompt=True,
    tokenize=True,
    return_tensors="pt",
)
inputs = {key: value.to("cuda") for key, value in inputs.items()}

with torch.inference_mode():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=256,
        do_sample=False,
        use_cache=True,
    )

response = processor.tokenizer.decode(
    output_ids[0, inputs["input_ids"].shape[1]:],
    skip_special_tokens=True,
)
print(response)