Upload processor

processing_taivisionlm.py

@@ -289,28 +289,28 @@ class TaiVisionProcessor(ProcessorMixin):
     
     
     
-if __name__ == '__main__':
-    from configuration_taivisionlm import TaiVisionLMConfig
-    import transformers
-    import torch
-    config = TaiVisionLMConfig.from_pretrained("./")
-    preprocessor = transformers.SiglipImageProcessor.from_pretrained("google/siglip-base-patch16-224")
-    preprocessor.image_seq_length = config.num_image_tokens
-    tokenizer = transformers.AutoTokenizer.from_pretrained("benchang1110/Taiwan-tinyllama-v1.0-chat")
-    processor = TaiVisionProcessor(tokenizer=tokenizer, image_processor=preprocessor)
-    processor.save_pretrained("./")
+# if __name__ == '__main__':
+#     from configuration_taivisionlm import TaiVisionLMConfig
+#     import transformers
+#     import torch
+#     config = TaiVisionLMConfig.from_pretrained("./")
+#     preprocessor = transformers.SiglipImageProcessor.from_pretrained("google/siglip-base-patch16-224")
+#     preprocessor.image_seq_length = config.num_image_tokens
+#     tokenizer = transformers.AutoTokenizer.from_pretrained("benchang1110/Taiwan-tinyllama-v1.0-chat")
+#     processor = TaiVisionProcessor(tokenizer=tokenizer, image_processor=preprocessor)
+#     processor.save_pretrained("./")
     
-    from PIL import Image
-    import requests
-    processor = TaiVisionProcessor.from_pretrained("./")
-    url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg"
-    image = Image.open(requests.get(url, stream=True).raw).convert("RGB")
-    text = "Hello< what is your name?"
-    suffix = "I am fine, thank you."
-    inputs = processor(text=text,suffix=suffix,images=image, return_tensors="pt",padding="max_length",max_length=512)
-    print(inputs['attention_mask'].shape)
-    print(inputs['input_ids'].shape)
-    print(inputs['token_type_ids'].shape)
-    # print number of 0 in token_type_ids
-    print(torch.sum(inputs['token_type_ids']==0))
-    print(inputs)
+#     from PIL import Image
+#     import requests
+#     processor = TaiVisionProcessor.from_pretrained("./")
+#     url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg"
+#     image = Image.open(requests.get(url, stream=True).raw).convert("RGB")
+#     text = "Hello< what is your name?"
+#     suffix = "I am fine, thank you."
+#     inputs = processor(text=text,suffix=suffix,images=image, return_tensors="pt",padding="max_length",max_length=512)
+#     print(inputs['attention_mask'].shape)
+#     print(inputs['input_ids'].shape)
+#     print(inputs['token_type_ids'].shape)
+#     # print number of 0 in token_type_ids
+#     print(torch.sum(inputs['token_type_ids']==0))
+#     print(inputs)