given the following code for model factory. can you adapt the llama 3.2 11b vision in order to be fully integrated in my pipeline?
this is the model facotry classes:
from abc import ABC, abstractmethod
from typing import List
import torch
from PIL import Image
from utils.utils import apply_prompt_template, EosListStoppingCriteria
from utils.processing_llavagemma import LlavaGemmaProcessor 
from qwen_vl_utils import process_vision_info
from transformers import (
    VipLlavaForConditionalGeneration,
    AutoProcessor,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    LlavaForConditionalGeneration,
    BlipForConditionalGeneration,
    BlipProcessor,
    AutoModelForCausalLM,
    pipeline,
    LlavaNextProcessor,
    LlavaNextForConditionalGeneration,
    AutoTokenizer,
    CLIPImageProcessor,
    AutoModelForVision2Seq,
    AutoImageProcessor,
    LlavaForConditionalGeneration,
    Qwen2VLForConditionalGeneration,
    MllamaForConditionalGeneration
    
)

class KeywordExtractor(ABC):
    """Abstract base class for keyword extraction models."""

    @abstractmethod
    def extract_keywords(self, image_path: str, prompt: str) -> List[str]:
        """Extract keywords from an image and prompt."""
        pass

class VipLlavaKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = VipLlavaForConditionalGeneration.from_pretrained(
            "llava-hf/vip-llava-7b-hf",
            device_map=self.device,
            torch_dtype=torch.float16,
        )
        self.processor = AutoProcessor.from_pretrained("llava-hf/vip-llava-7b-hf")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path)
        if question:
            prompt = prompt.format(question)
        inputs = self.processor(text=prompt, images=image, return_tensors="pt").to(self.device, torch.float16)
        # _outputs_ = self.model.generate(
        #     **inputs,
        #     do_sample=False,
        #     num_beams=5,
        #     #max_length=256,
        #     min_length=1,
        #     top_p=0.9,
        #     repetition_penalty=2.0,
        #     length_penalty=0.5,
        #     temperature=1,
        #     max_new_tokens=20
        # )
        #outputs = self.model.generate(**inputs, max_new_tokens=20)

        generate_ids = self.model.generate(**inputs, max_new_tokens=20)
        #keywords = self.processor.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=False)
        keywords = self.processor.decode(generate_ids[0][len(inputs["input_ids"][0]):], skip_special_tokens=True)

        return keywords#.split()

class InstructBlipKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = InstructBlipForConditionalGeneration.from_pretrained(
            "Salesforce/instructblip-vicuna-7b",
            torch_dtype=torch.float16,
            device_map=device
        )
        self.processor = InstructBlipProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)
        inputs = self.processor(images=image, text=prompt, return_tensors="pt").to(self.device, torch.float16)
        _outputs = self.model.generate(
            **inputs,
            do_sample=False,
            num_beams=5,
            #max_length=256,
            min_length=1,
            top_p=0.9,
            repetition_penalty=2.0,
            length_penalty=0.5,
            temperature=1,
            max_new_tokens=20
        )
        outputs = self.model.generate(**inputs, max_new_tokens=20)
        #keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)[0].strip().split()
        #return keywords.split()

        #keywords = self.processor.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=False)
        keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)
        return keywords#.split()

class LlavaKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf", device_map="auto", torch_dtype=torch.float16)
        self.processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path)
        if question:
            prompt = prompt.format(question)
        inputs = self.processor(text=prompt, images=image, return_tensors="pt").to(self.device, torch.float16)
        outputs_ = self.model.generate(
            **inputs,
            do_sample=False,
            num_beams=5,
            #max_length=256,
            min_length=1,
            top_p=0.9,
            repetition_penalty=2.0,
            length_penalty=0.5,
            temperature=1,
            max_new_tokens=20
        )
        outputs = self.model.generate(**inputs, max_new_tokens=20)

        
        generate_ids = self.model.generate(**inputs, max_new_tokens=20)
        #keywords = self.processor.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=False)
        #keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)
        keywords = self.processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        return keywords


class LlavaI2TKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.pipe = pipeline("image-to-text", model="llava-hf/llava-1.5-7b-hf", device_map=self.device, torch_dtype=torch.float16)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path)
        if question:
            prompt = prompt.format(question)
        outputs = self.pipe(image, prompt=prompt, generate_kwargs={"max_new_tokens": 200})
        return outputs


class LlavaMistralKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = LlavaNextForConditionalGeneration.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf",
                                                                       torch_dtype=torch.float16,
                                                                       low_cpu_mem_usage=True,
                                                                       device_map=device,
                                                                       # use_flash_attention_2=True,
                                                                       # load_in_4bit=True,
                                                                       )
        self.processor = LlavaNextProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path)
        if question:
            prompt = prompt.format(question)
        inputs = self.processor(text=prompt, images=image, return_tensors="pt").to(self.device, torch.float16)
        outputs_ = self.model.generate(
            **inputs,
            do_sample=False,
            num_beams=5,
            # max_length=256,
            min_length=1,
            top_p=0.9,
            repetition_penalty=2.0,
            length_penalty=0.5,
            temperature=1,
            max_new_tokens=20
        )
        outputs = self.model.generate(**inputs, max_new_tokens=100)

        generate_ids = self.model.generate(**inputs, max_new_tokens=100)
        keywords = self. processor.decode(generate_ids[0], skip_special_tokens=True)
        #keywords = self.processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        return keywords

class LlavaMistralI2TKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.pipe = pipeline("image-to-text", model="llava-hf/llava-v1.6-mistral-7b-hf", device_map=self.device, torch_dtype=torch.float16)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path)
        if question:
            prompt = prompt.format(question)
        outputs = self.pipe(image, prompt=prompt, generate_kwargs={"max_new_tokens": 200})
        return outputs


class BlipICLargeKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = BlipForConditionalGeneration.from_pretrained(
            "Salesforce/blip-image-captioning-large",
            #torch_dtype=torch.float16,
            device_map=device
        )
        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)
        inputs = self.processor(images=image, text=prompt, return_tensors="pt").to(self.device) #, torch.float16)
        _outputs = self.model.generate(
            **inputs,
            do_sample=False,
            num_beams=5,
            #max_length=256,
            min_length=1,
            top_p=0.9,
            repetition_penalty=2.0,
            length_penalty=0.5,
            temperature=1,
            max_new_tokens=20
        )
        outputs = self.model.generate(**inputs, max_new_tokens=20)#, max_new_tokens=20)
        #keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)[0].strip().split()
        #return keywords.split()

        #keywords = self.processor.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=False)
        keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)
        return keywords#.split()

class GitBaseICKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-coco",
            #torch_dtype=torch.float16,
            device_map=device,
        )
        self.processor = AutoProcessor.from_pretrained("microsoft/git-base-coco")

    def extract_keywords(self, image_path: str, prompt="", question="") -> List[str]:
        image = Image.open(image_path).convert("RGB")
        inputs = self.processor(images=image, return_tensors="pt").pixel_values.to(self.device) #, torch.float16)
        outputs = self.model.generate(pixel_values=inputs, max_length=50)
        keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)[0]

        return keywords#.split()




class GitBaseVQAKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.processor = AutoProcessor.from_pretrained("microsoft/git-base-textvqa")
        self.model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-textvqa")
        self.model.to(self.device)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)

        pixel_values = self.processor(images=image, return_tensors="pt").pixel_values  # No need to move to device
        input_ids = self.processor(text=prompt, add_special_tokens=False).input_ids
        input_ids = [self.processor.tokenizer.cls_token_id] + input_ids
        input_ids = torch.tensor(input_ids).unsqueeze(0).to(self.device)

        # outputs = self.model.generate(pixel_values=pixel_values.to(self.device), input_ids=input_ids, max_length=50)  # Move pixel_values to device
        outputs = self.model.generate(pixel_values=pixel_values.to(self.device), input_ids=input_ids, max_new_tokens=100)  # Move pixel_values to device
        keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)
        return keywords


class BunnyKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = AutoModelForCausalLM.from_pretrained(
            'BAAI/Bunny-v1_0-3B',
            torch_dtype=torch.float16,
            device_map=device,
            trust_remote_code=True)

        self.tokenizer = AutoTokenizer.from_pretrained(
            'BAAI/Bunny-v1_0-3B',
            trust_remote_code=True)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        torch.set_default_device(self.device)
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)
        text_chunks = [self.tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
        input_ids = torch.tensor(text_chunks[0] + [-200] + text_chunks[1], dtype=torch.long).unsqueeze(0)
        image_tensor = self.model.process_images([image], self.model.config).to(dtype=self.model.dtype)

        # inputs = self.processor(images=image, text=prompt, return_tensors="pt").to(self.device) #, torch.float16)
        # outputs = self.model.generate(**inputs, max_new_tokens=20)#, max_new_tokens=20)

        output_ids = self.model.generate(
            input_ids,
            images=image_tensor,
            max_new_tokens=100,
            use_cache=True)[0]

        keywords = self.tokenizer.decode(output_ids[input_ids.shape[1]:], skip_special_tokens=True).strip()
        return keywords#.split()

class Moondream2KeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = AutoModelForCausalLM.from_pretrained(
            "vikhyatk/moondream2",
            torch_dtype=torch.float16,
            device_map=device,
            trust_remote_code=True)


        self.tokenizer = AutoTokenizer.from_pretrained(
            "vikhyatk/moondream2",
            trust_remote_code=True)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        torch.set_default_device(self.device)
        image = Image.open(image_path).convert("RGB")
        enc_image = self.model.encode_image(image)
        if question:
            prompt = prompt.format(question)

        if "<image>" in prompt:
            prompt = prompt.replace("<image>", "")
        else:
            prompt

        keywords = self.model.answer_question(enc_image, prompt, self.tokenizer)
        return keywords#.split()
    

class IntelLlavaGemmaKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = LlavaForConditionalGeneration.from_pretrained("Intel/llava-gemma-2b", device_map="auto")#, torch_dtype=torch.float16)
        self.processor = LlavaGemmaProcessor(
                            tokenizer=AutoTokenizer.from_pretrained("Intel/llava-gemma-2b"),
                            image_processor=CLIPImageProcessor.from_pretrained("Intel/llava-gemma-2b"),

)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path)
        if question:
            prompt = prompt.format(question)

        prompt = self.processor.tokenizer.apply_chat_template(
                    [{'role': 'user', 'content': prompt}],
                    tokenize=False,
                    add_generation_prompt=True
                )
        
        inputs = self.processor(text=prompt, images=image, return_tensors="pt").to(self.device)#, torch.float16)
        # inputs = {k: v.to('cuda') for k, v in inputs.items()}
 
        generate_ids = self.model.generate(**inputs, max_new_tokens=50)

        keywords = self.processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        return keywords


class QwenVLKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = AutoModelForCausalLM.from_pretrained(
            "Qwen/Qwen-VL-Chat-Int4",
            torch_dtype=torch.float16,
            device_map=device,
            trust_remote_code=True).eval()

        self.tokenizer = AutoTokenizer.from_pretrained(
            'Qwen/Qwen-VL-Chat-Int4',
            trust_remote_code=True)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        torch.set_default_device(self.device)
        # image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)
        # text_chunks = [self.tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
        # input_ids = torch.tensor(text_chunks[0] + [-200] + text_chunks[1], dtype=torch.long).unsqueeze(0)
        # image_tensor = self.model.process_images([image], self.model.config).to(dtype=self.model.dtype)

        query = self.tokenizer.from_list_format([
            {'image': image_path},
            {'text': prompt},
        ])

        response, history = self.model.chat(self.tokenizer, query=query, history=None)
        return response
        #
        #
        # keywords = self.tokenizer.decode(output_ids[input_ids.shape[1]:], skip_special_tokens=True).strip()
        # return keywords#.split()

class Qwen2VL7BKeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = Qwen2VLForConditionalGeneration.from_pretrained(
            "Qwen/Qwen2-VL-7B-Instruct",
            torch_dtype="auto",
            device_map="auto"
        ).eval()

        self.processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        torch.set_default_device(self.device)
        # image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)

        messages = [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image",
                        "image": image_path,
                    },
                    {"type": "text", "text": prompt},
                ],
            }
        ]

        text = self.processor.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True
        )
        image_inputs, video_inputs = process_vision_info(messages)
        inputs = self.processor(
            text=[text],
            images=image_inputs,
            videos=video_inputs,
            padding=True,
            return_tensors="pt",
        )
        inputs = inputs.to(self.device)

        generated_ids = self.model.generate(**inputs, max_new_tokens=128)
        generated_ids_trimmed = [
            out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
        ]
        output_text = self.processor.batch_decode(
            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
        )
        return output_text[0]


class Idefics2KeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = AutoModelForVision2Seq.from_pretrained(
    "HuggingFaceM4/idefics2-8b",
            torch_dtype=torch.float16,
            device_map=device
        )
        self.processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics2-8b")

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)
        inputs = self.processor(images=image, text=prompt, return_tensors="pt").to(self.device, torch.float16)

        outputs = self.model.generate(**inputs, max_new_tokens=200)
        keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)
        return keywords


class Blip3KeywordExtractor(KeywordExtractor):

    def __init__(self, device: str):
        self.device = torch.device(device)
        self.tokenizer = AutoTokenizer.from_pretrained("Salesforce/blip3-phi3-mini-instruct-r-v1", trust_remote_code=True, use_fast=False, legacy=False)
        self.model = AutoModelForVision2Seq.from_pretrained(
    "Salesforce/blip3-phi3-mini-instruct-r-v1",
            torch_dtype=torch.float16,
            device_map=device,
            trust_remote_code=True
        )
        self.processor = AutoImageProcessor.from_pretrained("Salesforce/blip3-phi3-mini-instruct-r-v1", trust_remote_code=True)

    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        self.tokenizer = self.model.update_special_tokens(self.tokenizer)
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)

        prompt = apply_prompt_template(prompt)
        language_inputs = self.tokenizer([prompt], return_tensors="pt").to(self.device, torch.float16)
        inputs = self.processor(images=image, return_tensors="pt", image_aspect_ratio='anyres').to(self.device, torch.float16)
        inputs.update(language_inputs)
        inputs = {name: tensor.cuda() for name, tensor in inputs.items()}
        outputs = self.model.generate(**inputs,image_size=[image.size],
                                pad_token_id=self.tokenizer.pad_token_id,
                                do_sample=False, max_new_tokens=768, top_p=None, num_beams=1,
                                stopping_criteria = [EosListStoppingCriteria()],
                                )
        keywords = self.tokenizer.decode(outputs[0], skip_special_tokens=True).split("
")[0]
        return keywords

class LlavaLlama3KeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = LlavaForConditionalGeneration.from_pretrained(
    "xtuner/llava-llama-3-8b-v1_1-transformers",
            torch_dtype=torch.float16,
            low_cpu_mem_usage=True, #True
            device_map=device
        )
        self.processor = AutoProcessor.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers")


    def extract_keywords(self, image_path: str, prompt: str, question: str) -> List[str]:
        image = Image.open(image_path).convert("RGB")
        if question:
            prompt = prompt.format(question)

        prompt = (f"
user
\n\n{prompt}
"
          "
assistant
\n\n")
        inputs = self.processor(images=image, text=prompt, return_tensors="pt").to(self.device, torch.float16)

        outputs = self.model.generate(**inputs, max_new_tokens=200, do_sample=False)
        keywords = self.processor.batch_decode(outputs, skip_special_tokens=True)
        return keywords

class Phi3KeywordExtractor(KeywordExtractor):
    def __init__(self, device: str):
        self.device = torch.device(device)
        self.model = AutoModelForCausalLM.from_pretrained("microsoft/Phi-3.5-vision-instruct",
            torch_dtype=torch.bfloat16,
            device_map=device,
            _attn_implementation='flash_attention_2',    
            trust_remote_code=True
        )
        self.processor = AutoProcessor.from_pretrained("microsoft/Phi-3.5-vision-instruct", trust_remote_code=True, num_crops=16)

    def extract_keywords(self, image_path: str, prompt="", question="") -> List[str]:
        user_prompt = '
\n'
        assistant_prompt = '
\n'
        prompt_suffix = "
\n"
        full_prompt = f"{user_prompt}
\n{prompt}{prompt_suffix}{assistant_prompt}"
        image = Image.open(image_path).convert("RGB")
        # image = Image.fromarray(image_path).convert("RGB")
        inputs = self.processor(images=image, text=full_prompt,  return_tensors="pt").to(self.device)
        outputs = self.model.generate(**inputs, max_new_tokens=200, do_sample=False, eos_token_id=self.processor.tokenizer.eos_token_id)
        outputs = outputs[:, inputs['input_ids'].shape[1]:]
        keywords = self.processor.batch_decode(outputs,   skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        #print(keywords)
        return keywords



this isgeneral model card for llama 3.2

import requests
import torch
from PIL import Image
from transformers import MllamaForConditionalGeneration, AutoProcessor

model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"

model = MllamaForConditionalGeneration.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)
processor = AutoProcessor.from_pretrained(model_id)

url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg"
image = Image.open(requests.get(url, stream=True).raw)

messages = [
    {"role": "user", "content": [
        {"type": "image"},
        {"type": "text", "text": "If I had to write a haiku for this one, it would be: "}
    ]}
]
input_text = processor.apply_chat_template(messages, add_generation_prompt=True)
inputs = processor(
    image,
    input_text,
    add_special_tokens=False,
    return_tensors="pt"
).to(model.device)

output = model.generate(**inputs, max_new_tokens=30)
print(processor.decode(output[0]))