update

multipurpose_chatbot/engines/image_processing_llava_next.py

@@ -434,6 +434,8 @@ class LlavaNextImageProcessor(BaseImageProcessor):
             for patch in patches
         ]
 
+        print(f'patches: {len(patches)=} given {image_size}, {best_resolution}')
+
         resized_original_image = resize(
             image,
             size=size,
@@ -606,6 +608,7 @@ class LlavaNextImageProcessor(BaseImageProcessor):
 
         if concat_images:
             # image_num_patches = [len(x) for x in new_images]
+            print(f'{len(new_images)}')
             pixel_values = np.concatenate(new_images, axis=0)
             data = {
                 "pixel_values": pixel_values,

multipurpose_chatbot/engines/sealava16_transformers_engine.py

@@ -162,25 +162,28 @@ class SeaLlava16Engine(TransformersEngine):
         sys.path.append(CODE_PATH)
 
 
-        # from transformers.models.llava_next.modeling_llava_next import LlavaNextForConditionalGeneration
-        # from transformers.models.llava_next.processing_llava_next import LlavaNextProcessor
         from .modeling_sealava16 import LlavaNextForConditionalGeneration
         from .image_processing_llava_next import LlavaNextImageProcessor
         from .processing_llava_next import LlavaNextProcessor
         model_path = MODEL_PATH
         print(f'Loading model from {model_path}')
+        
+        # default
+        from transformers.models.llava_next.modeling_llava_next import LlavaNextForConditionalGeneration
+        from transformers.models.llava_next.processing_llava_next import LlavaNextProcessor
+        self._processor = LlavaNextProcessor.from_pretrained(model_path)
+        self._model = LlavaNextForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.bfloat16, device_map="cuda").eval()
+        print(f'Loading llava1.6 from default code')
 
-        print(f'model_path={model_path}')
-        if os.path.exists(f"{model_path}/pytorch_model_fsdp.bin") and not os.path.exists(f"{model_path}/pytorch_model.bin"):
-            os.symlink("pytorch_model_fsdp.bin", f"{model_path}/pytorch_model.bin")
+        # /custom
 
-        # self._processor = LlavaNextProcessor.from_pretrained(model_path)
-        self._tokenizer = AutoTokenizer.from_pretrained(model_path)
-        self._image_processor = LlavaNextImageProcessor(model_path)
-        self._processor = LlavaNextProcessor(image_processor=self._image_processor, tokenizer=self._tokenizer)
-        self._model = LlavaNextForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.bfloat16, device_map="cuda").eval()
+        # self._tokenizer = AutoTokenizer.from_pretrained(model_path)
+        # self._image_processor = LlavaNextImageProcessor(model_path)
+        # self._processor = LlavaNextProcessor(image_processor=self._image_processor, tokenizer=self._tokenizer)
+        # self._model = LlavaNextForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.bfloat16, device_map="cuda").eval()
         print(f'Loading llava1.6 from custom code')
-        print(self._image_processor)
+        
+        print(self._processor.image_processor)
         print(self._tokenizer)
         print(self._processor)