Update README.md

README.md

@@ -7,3 +7,109 @@ tags: []
 
 WanDB https://wandb.ai/huseinzol05/vision-tinyllama?workspace=user-huseinzol05
 
+## how-to
+
+```python
+from modeling_vision import MM_LLMs, MM_LLMs_Config
+from transformers import AutoTokenizer, AutoProcessor
+from PIL import Image
+import requests
+
+def prepare_dataset(messages, images: List[str] = None):
+    if images is not None:
+        images = [Image.open(f).convert('RGB') for f in images]
+        image_output = image_processor(images=images, return_tensors='pt')['pixel_values']
+    else:
+        image_output = None
+    
+    prompt = tokenizer.apply_chat_template(messages, tokenize = False)
+    outputs = tokenizer(
+                    prompt,
+                    return_tensors='pt',
+                    return_overflowing_tokens=False,
+                    return_length=False)
+
+    outputs['images'] = image_output
+    outputs['image_index'] = torch.tensor([0] * len(outputs['images']))
+    outputs['image_starts'] = torch.tensor([tokenizer.convert_tokens_to_ids('<image>')] * len(outputs['images']))
+    return outputs
+
+model = MM_LLMs.from_pretrained(
+    'mesolitica/malaysian-tinyllama-1.1b-siglip-base-384-vision',
+    flash_attention = True,
+    dtype = torch.bfloat16,
+    torch_dtype = torch.bfloat16
+)
+_ = model.cuda()
+
+image_processor = AutoProcessor.from_pretrained('google/siglip-base-patch16-384')
+tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-siglip-base-384-vision')
+
+with open('Persian-cat-breed.jpg', 'wb') as fopen:
+    fopen.write(requests.get('https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg').content)
+
+with open('nasi-goreng-1-23.jpg', 'wb') as fopen:
+    fopen.write(requests.get('https://www.jocooks.com/wp-content/uploads/2023/09/nasi-goreng-1-23.jpg').content)
+
+messages = [
+    {'role': 'user', 'content': '<image> </image> ini gambar apa'},
+]
+outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg'])
+outputs['images'] = outputs['images'].type(model.dtype)
+for k in outputs.keys():
+    if outputs[k] is not None:
+        outputs[k] = outputs[k].cuda()
+
+with torch.no_grad():
+    model_inputs = model.prepare_inputs_for_generation(**outputs)
+r = model_inputs.pop('input_ids', None)
+
+generate_kwargs = dict(
+    model_inputs,
+    max_new_tokens=300,
+    top_p=0.95,
+    top_k=50,
+    temperature=0.1,
+    do_sample=True,
+    num_beams=1,
+)
+
+r = model.llm.generate(**generate_kwargs)
+print(tokenizer.decode(r[0]))
+```
+
+```
+<s>Imej itu menunjukkan seekor kucing putih yang comel duduk di atas sofa hitam.</s>
+```
+
+```python
+messages = [
+    {'role': 'user', 'content': '<image> </image> <image> </image> apa kaitan 2 gambar ni'},
+]
+outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg', 'nasi-goreng-1-23.jpg'])
+outputs['images'] = outputs['images'].type(model.dtype)
+for k in outputs.keys():
+    if outputs[k] is not None:
+        outputs[k] = outputs[k].cuda()
+
+with torch.no_grad():
+    model_inputs = model.prepare_inputs_for_generation(**outputs)
+r = model_inputs.pop('input_ids', None)
+
+generate_kwargs = dict(
+    model_inputs,
+    max_new_tokens=300,
+    top_p=0.95,
+    top_k=50,
+    temperature=0.1,
+    do_sample=True,
+    num_beams=1,
+)
+
+r = model.llm.generate(**generate_kwargs)
+print(tokenizer.decode(r[0]))
+```
+
+```
+<s>Tiada hubungan yang jelas antara gambar 1 (anak kucing putih duduk di atas sofa) dan gambar 2 (foto penutup mangkuk mi telur dengan nasi dan cili). Gambar pertama ialah imej haiwan, manakala gambar kedua ialah imej makanan. Mereka tergolong dalam kategori yang berbeza dan tidak mempunyai hubungan antara satu sama lain.</s>
+```