Update README.md

README.md

@@ -119,6 +119,100 @@ The image below shows reproductions of two representative pages of the scientifi
 
 ![image/png](https://cdn-uploads.huggingface.co/production/uploads/623ce1c6b66fedf374859fe7/qHURSBRWEDgHy4o56escN.png)
 
+## Fine-tuning
+
+
+Load base model 
+
+```python
+model_id = "microsoft/Phi-3-vision-128k-instruct" 
+
+model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda", trust_remote_code=True, torch_dtype="auto")
+
+processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True) 
+```
+
+Define FT_repo_id to push on HF hub/save model:
+```
+FT_repo_id='xxxxx/' #<repo_ID>
+```
+
+```
+from datasets import load_dataset
+
+train_dataset = load_dataset("lamm-mit/Cephalo-Wikipedia-Materials", split="train")
+```
+
+```python
+import random
+
+class MyDataCollator:
+    def __init__(self, processor):
+        self.processor = processor
+
+    def __call__(self, examples):
+        texts = []
+        images = []
+        for example in examples:
+            image = example["image"]
+            question = example["query"] 
+            answer = example["answer"]            
+            messages = [ {
+                            "role": "user",  "content": '<|image_1|>\n'+question},
+                           {"role": "assistant", "content": f"{answer}"}, ]
+                
+            text = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
+                
+            images.append(image)
+             
+        batch = processor(text=text, images=[image], return_tensors="pt", padding=True
+            
+        labels = batch["input_ids"].clone() 
+        labels[labels <0] = -100 
+
+        batch["labels"] = labels
+
+        return batch
+
+data_collator = MyDataCollator(processor)
+```
+Then set up trainer, and train:
+```python
+from transformers import TrainingArguments, Trainer
+
+optim = "paged_adamw_8bit"
+
+training_args = TrainingArguments(
+    num_train_epochs=2,
+    per_device_train_batch_size=1,
+    #per_device_eval_batch_size=4,
+    gradient_accumulation_steps=4,
+    warmup_steps=250,
+    learning_rate=1e-5,
+    weight_decay=0.01,
+    logging_steps=25,
+    output_dir="output_training",
+    optim=optim,
+    save_strategy="steps",
+    save_steps=1000,
+    save_total_limit=16,
+    #fp16=True,
+    bf16=True,  
+    push_to_hub_model_id=FT_repo_id,
+    remove_unused_columns=False,
+    report_to="none",
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    data_collator=data_collator,
+    train_dataset=train_dataset,
+)
+
+trainer.train()
+```
+
 ## Citation
 
 Please cite as: