Spaces:
Runtime error
Runtime error
Duplicate from kasun/comparing-captioning-models
Browse filesCo-authored-by: kasun <kasun@users.noreply.huggingface.co>
- .gitattributes +34 -0
- README.md +13 -0
- app.py +125 -0
- requirements.txt +5 -0
.gitattributes
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: Comparing Captioning Models
|
3 |
+
emoji: 🔥
|
4 |
+
colorFrom: yellow
|
5 |
+
colorTo: pink
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 3.15.0
|
8 |
+
app_file: app.py
|
9 |
+
pinned: false
|
10 |
+
duplicated_from: kasun/comparing-captioning-models
|
11 |
+
---
|
12 |
+
|
13 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
app.py
ADDED
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
from transformers import AutoProcessor, BlipForConditionalGeneration, AutoModelForCausalLM, AutoImageProcessor, VisionEncoderDecoderModel, AutoTokenizer
|
3 |
+
|
4 |
+
# from transformers import AutoProcessor, AutoTokenizer, AutoImageProcessor, AutoModelForCausalLM, BlipForConditionalGeneration, Blip2ForConditionalGeneration, VisionEncoderDecoderModel
|
5 |
+
import torch
|
6 |
+
import open_clip
|
7 |
+
|
8 |
+
from huggingface_hub import hf_hub_download
|
9 |
+
|
10 |
+
torch.hub.download_url_to_file('http://images.cocodataset.org/val2017/000000039769.jpg', 'cats.jpg')
|
11 |
+
torch.hub.download_url_to_file('https://huggingface.co/datasets/nielsr/textcaps-sample/resolve/main/stop_sign.png', 'stop_sign.png')
|
12 |
+
torch.hub.download_url_to_file('https://cdn.openai.com/dall-e-2/demos/text2im/astronaut/horse/photo/0.jpg', 'astronaut.jpg')
|
13 |
+
|
14 |
+
git_processor_base = AutoProcessor.from_pretrained("microsoft/git-base-coco")
|
15 |
+
git_model_base = AutoModelForCausalLM.from_pretrained("microsoft/git-base-coco")
|
16 |
+
|
17 |
+
# git_processor_large_coco = AutoProcessor.from_pretrained("microsoft/git-large-coco")
|
18 |
+
# git_model_large_coco = AutoModelForCausalLM.from_pretrained("microsoft/git-large-coco")
|
19 |
+
|
20 |
+
# git_processor_large_textcaps = AutoProcessor.from_pretrained("microsoft/git-large-r-textcaps")
|
21 |
+
# git_model_large_textcaps = AutoModelForCausalLM.from_pretrained("microsoft/git-large-r-textcaps")
|
22 |
+
|
23 |
+
blip_processor_base = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
|
24 |
+
blip_model_base = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
|
25 |
+
|
26 |
+
# blip_processor_large = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
|
27 |
+
# blip_model_large = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
|
28 |
+
|
29 |
+
# blip2_processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-2.7b")
|
30 |
+
# blip2_model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b", torch_dtype=torch.float16)
|
31 |
+
|
32 |
+
# blip2_processor_8_bit = AutoProcessor.from_pretrained("Salesforce/blip2-opt-6.7b")
|
33 |
+
# blip2_model_8_bit = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b", device_map="auto", load_in_8bit=True)
|
34 |
+
|
35 |
+
# vitgpt_processor = AutoImageProcessor.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
|
36 |
+
# vitgpt_model = VisionEncoderDecoderModel.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
|
37 |
+
# vitgpt_tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
|
38 |
+
|
39 |
+
# coca_model, _, coca_transform = open_clip.create_model_and_transforms(
|
40 |
+
# model_name="coca_ViT-L-14",
|
41 |
+
# pretrained="mscoco_finetuned_laion2B-s13B-b90k"
|
42 |
+
# )
|
43 |
+
|
44 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
45 |
+
|
46 |
+
git_model_base.to(device)
|
47 |
+
blip_model_base.to(device)
|
48 |
+
# git_model_large_coco.to(device)
|
49 |
+
# git_model_large_textcaps.to(device)
|
50 |
+
# blip_model_large.to(device)
|
51 |
+
# vitgpt_model.to(device)
|
52 |
+
# coca_model.to(device)
|
53 |
+
# blip2_model.to(device)
|
54 |
+
|
55 |
+
def generate_caption(processor, model, image, tokenizer=None, use_float_16=False):
|
56 |
+
inputs = processor(images=image, return_tensors="pt").to(device)
|
57 |
+
|
58 |
+
if use_float_16:
|
59 |
+
inputs = inputs.to(torch.float16)
|
60 |
+
|
61 |
+
generated_ids = model.generate(pixel_values=inputs.pixel_values, max_length=50)
|
62 |
+
|
63 |
+
if tokenizer is not None:
|
64 |
+
generated_caption = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
65 |
+
else:
|
66 |
+
generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
67 |
+
|
68 |
+
return generated_caption
|
69 |
+
|
70 |
+
|
71 |
+
def generate_caption_coca(model, transform, image):
|
72 |
+
im = transform(image).unsqueeze(0).to(device)
|
73 |
+
with torch.no_grad(), torch.cuda.amp.autocast():
|
74 |
+
generated = model.generate(im, seq_len=20)
|
75 |
+
return open_clip.decode(generated[0].detach()).split("<end_of_text>")[0].replace("<start_of_text>", "")
|
76 |
+
|
77 |
+
|
78 |
+
def generate_captions(image):
|
79 |
+
caption_git_base = generate_caption(git_processor_base, git_model_base, image)
|
80 |
+
|
81 |
+
# caption_git_large_coco = generate_caption(git_processor_large_coco, git_model_large_coco, image)
|
82 |
+
|
83 |
+
# caption_git_large_textcaps = generate_caption(git_processor_large_textcaps, git_model_large_textcaps, image)
|
84 |
+
|
85 |
+
caption_blip_base = generate_caption(blip_processor_base, blip_model_base, image)
|
86 |
+
|
87 |
+
# caption_blip_large = generate_caption(blip_processor_large, blip_model_large, image)
|
88 |
+
|
89 |
+
# caption_vitgpt = generate_caption(vitgpt_processor, vitgpt_model, image, vitgpt_tokenizer)
|
90 |
+
|
91 |
+
# caption_coca = generate_caption_coca(coca_model, coca_transform, image)
|
92 |
+
|
93 |
+
# caption_blip2 = generate_caption(blip2_processor, blip2_model, image, use_float_16=True).strip()
|
94 |
+
|
95 |
+
# caption_blip2_8_bit = generate_caption(blip2_processor_8_bit, blip2_model_8_bit, image, use_float_16=True).strip()
|
96 |
+
|
97 |
+
# return caption_git_large_coco, caption_git_large_textcaps, caption_blip_large, caption_coca, caption_blip2_8_bit
|
98 |
+
return caption_git_base, caption_blip_base
|
99 |
+
|
100 |
+
|
101 |
+
|
102 |
+
examples = [["cats.jpg"], ["stop_sign.png"], ["astronaut.jpg"]]
|
103 |
+
# outputs = [gr.outputs.Textbox(label="Caption generated by GIT-large fine-tuned on COCO"), gr.outputs.Textbox(label="Caption generated by GIT-large fine-tuned on TextCaps"), gr.outputs.Textbox(label="Caption generated by BLIP-large"), gr.outputs.Textbox(label="Caption generated by CoCa"), gr.outputs.Textbox(label="Caption generated by BLIP-2 OPT 6.7b")]
|
104 |
+
outputs = [
|
105 |
+
gr.outputs.Textbox(label="Caption generated by GIT-base fine-tuned on COCO"),
|
106 |
+
# gr.outputs.Textbox(label="Caption generated by GIT-large fine-tuned on COCO"),
|
107 |
+
# gr.outputs.Textbox(label="Caption generated by GIT-large fine-tuned on TextCaps"),
|
108 |
+
gr.outputs.Textbox(label="Caption generated by BLIP-base"),
|
109 |
+
# gr.outputs.Textbox(label="Caption generated by BLIP-large"),
|
110 |
+
# gr.outputs.Textbox(label="Caption generated by vitgpt")
|
111 |
+
]
|
112 |
+
|
113 |
+
title = "Interactive demo: comparing image captioning models"
|
114 |
+
description = "Gradio Demo to compare GIT, BLIP, CoCa, and BLIP-2, 4 state-of-the-art vision+language models. To use it, simply upload your image and click 'submit', or click one of the examples to load them. Read more at the links below."
|
115 |
+
article = "<p style='text-align: center'><a href='https://huggingface.co/docs/transformers/main/model_doc/blip' target='_blank'>BLIP docs</a> | <a href='https://huggingface.co/docs/transformers/main/model_doc/git' target='_blank'>GIT docs</a></p>"
|
116 |
+
|
117 |
+
interface = gr.Interface(fn=generate_captions,
|
118 |
+
inputs=gr.inputs.Image(type="pil"),
|
119 |
+
outputs=outputs,
|
120 |
+
examples=examples,
|
121 |
+
title=title,
|
122 |
+
description=description,
|
123 |
+
article=article,
|
124 |
+
enable_queue=True)
|
125 |
+
interface.launch()
|
requirements.txt
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
git+https://github.com/huggingface/transformers.git@main
|
2 |
+
torch
|
3 |
+
scipy
|
4 |
+
open_clip_torch
|
5 |
+
|