Kiel-2.1-Omni

Kiel-2.1-Omni is a fully merged multimodal model fine-tuned from Janus-Pro-1B on a custom curriculum. It delivers dual-modality capabilities: high-fidelity Text-to-Image Generation and precise Multimodal Vision Understanding/Chat.

---

Deployment & Loading Instructions

Load in 16-Bit (Full Precision)

import torch
from janus.models import MultiModalityCausalLM

model = MultiModalityCausalLM.from_pretrained(
    "kiel2/Kiel-2.1-Omni",
    torch_dtype=torch.float16,
    device_map="auto",
    trust_remote_code=True
).eval()
Load in Memory-Efficient 4-Bit
Python
import torch
from janus.models import MultiModalityCausalLM
from transformers import BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True
)

model = MultiModalityCausalLM.from_pretrained(
    "kiel2/Kiel-2.1-Omni",
    quantization_config=quantization_config,
    device_map="auto",
    trust_remote_code=True
).eval()
Quickstart Inference Examples
1. Multimodal Image Understanding & Chat
Python
import torch
from PIL import Image
from janus.models import VLChatProcessor

# Load processor and tokenizer
processor = VLChatProcessor.from_pretrained("kiel2/Kiel-2.1-Omni", trust_remote_code=True)
tokenizer = processor.tokenizer
device = "cuda" if torch.cuda.is_available() else "cpu"

image = Image.open("path_to_image.jpg").convert("RGB")
query = "Describe the architectural style and lighting of this scene in detail."

conversation = [
    {"role": "<|User|>", "content": f"<image_placeholder>\n{query}"},
    {"role": "<|Assistant|>", "content": ""}
]

inputs = processor(
    conversations=[conversation],
    images=[image],
    padding=True,
    return_tensors="pt"
).to(device)

inputs_embeds = model.prepare_multimodal_embeds(**inputs)

outputs = model.language_model.generate(
    inputs_embeds=inputs_embeds,
    attention_mask=inputs.attention_mask,
    pad_token_id=tokenizer.eos_token_id,
    bos_token_id=tokenizer.bos_token_id,
    eos_token_id=tokenizer.eos_token_id,
    max_new_tokens=256,
    do_sample=True,
    temperature=0.4,
    top_p=0.9
)

response = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(response)
2. Text-to-Image Generation
Python
import torch
import numpy as np
from PIL import Image
from janus.models import VLChatProcessor

processor = VLChatProcessor.from_pretrained("kiel2/Kiel-2.1-Omni", trust_remote_code=True)
tokenizer = processor.tokenizer
device = "cuda" if torch.cuda.is_available() else "cpu"

prompt = "A majestic cyberpunk castle overlooking a neon-lit futuristic city at sunset, cinematic lighting"
conversation = [
    {"role": "<|User|>", "content": prompt},
    {"role": "<|Assistant|>", "content": ""}
]

sft_format = processor.apply_sft_template_for_multi_turn_prompts(
    conversations=conversation,
    sft_format=processor.sft_format,
    system_prompt=""
)
full_prompt = sft_format + processor.image_start_tag
input_ids = tokenizer.encode(full_prompt, return_tensors="pt").to(device)

# Generate image tokens autoregressively...