dhanushreddy29 nielsr HF staff commited on
Commit
2e5b227
0 Parent(s):

Duplicate from nielsr/comparing-captioning-models

Browse files

Co-authored-by: Niels Rogge <nielsr@users.noreply.huggingface.co>

Files changed (4) hide show
  1. .gitattributes +34 -0
  2. README.md +13 -0
  3. app.py +75 -0
  4. requirements.txt +2 -0
.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Comparing Captioning Models
3
+ emoji: 🔥
4
+ colorFrom: yellow
5
+ colorTo: pink
6
+ sdk: gradio
7
+ sdk_version: 3.15.0
8
+ app_file: app.py
9
+ pinned: false
10
+ duplicated_from: nielsr/comparing-captioning-models
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from transformers import AutoProcessor, AutoTokenizer, AutoImageProcessor, AutoModelForCausalLM, BlipForConditionalGeneration, VisionEncoderDecoderModel
3
+ import torch
4
+
5
+ torch.hub.download_url_to_file('http://images.cocodataset.org/val2017/000000039769.jpg', 'cats.jpg')
6
+ torch.hub.download_url_to_file('https://huggingface.co/datasets/nielsr/textcaps-sample/resolve/main/stop_sign.png', 'stop_sign.png')
7
+ torch.hub.download_url_to_file('https://cdn.openai.com/dall-e-2/demos/text2im/astronaut/horse/photo/0.jpg', 'astronaut.jpg')
8
+
9
+ git_processor_base = AutoProcessor.from_pretrained("microsoft/git-base-coco")
10
+ git_model_base = AutoModelForCausalLM.from_pretrained("microsoft/git-base-coco")
11
+
12
+ git_processor_large = AutoProcessor.from_pretrained("microsoft/git-large-coco")
13
+ git_model_large = AutoModelForCausalLM.from_pretrained("microsoft/git-large-coco")
14
+
15
+ blip_processor_base = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
16
+ blip_model_base = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
17
+
18
+ blip_processor_large = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
19
+ blip_model_large = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
20
+
21
+ vitgpt_processor = AutoImageProcessor.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
22
+ vitgpt_model = VisionEncoderDecoderModel.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
23
+ vitgpt_tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
24
+
25
+ device = "cuda" if torch.cuda.is_available() else "cpu"
26
+
27
+ git_model_base.to(device)
28
+ blip_model_base.to(device)
29
+ git_model_large.to(device)
30
+ blip_model_large.to(device)
31
+ vitgpt_model.to(device)
32
+
33
+ def generate_caption(processor, model, image, tokenizer=None):
34
+ inputs = processor(images=image, return_tensors="pt").to(device)
35
+
36
+ generated_ids = model.generate(pixel_values=inputs.pixel_values, max_length=50)
37
+
38
+ if tokenizer is not None:
39
+ generated_caption = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
40
+ else:
41
+ generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
42
+
43
+ return generated_caption
44
+
45
+
46
+ def generate_captions(image):
47
+ caption_git_base = generate_caption(git_processor_base, git_model_base, image)
48
+
49
+ caption_git_large = generate_caption(git_processor_large, git_model_large, image)
50
+
51
+ caption_blip_base = generate_caption(blip_processor_base, blip_model_base, image)
52
+
53
+ caption_blip_large = generate_caption(blip_processor_large, blip_model_large, image)
54
+
55
+ caption_vitgpt = generate_caption(vitgpt_processor, vitgpt_model, image, vitgpt_tokenizer)
56
+
57
+ return caption_git_base, caption_git_large, caption_blip_base, caption_blip_large, caption_vitgpt
58
+
59
+
60
+ examples = [["cats.jpg"], ["stop_sign.png"], ["astronaut.jpg"]]
61
+ outputs = [gr.outputs.Textbox(label="Caption generated by GIT-base"), gr.outputs.Textbox(label="Caption generated by GIT-large"), gr.outputs.Textbox(label="Caption generated by BLIP-base"), gr.outputs.Textbox(label="Caption generated by BLIP-large"), gr.outputs.Textbox(label="Caption generated by ViT+GPT-2")]
62
+
63
+ title = "Interactive demo: comparing image captioning models"
64
+ description = "Gradio Demo to compare GIT, BLIP and ViT+GPT2, 3 state-of-the-art vision+language models. To use it, simply upload your image and click 'submit', or click one of the examples to load them. Read more at the links below."
65
+ article = "<p style='text-align: center'><a href='https://huggingface.co/docs/transformers/main/model_doc/blip' target='_blank'>BLIP docs</a> | <a href='https://huggingface.co/docs/transformers/main/model_doc/git' target='_blank'>GIT docs</a></p>"
66
+
67
+ interface = gr.Interface(fn=generate_captions,
68
+ inputs=gr.inputs.Image(type="pil"),
69
+ outputs=outputs,
70
+ examples=examples,
71
+ title=title,
72
+ description=description,
73
+ article=article,
74
+ enable_queue=True)
75
+ interface.launch(debug=True)
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ git+https://github.com/huggingface/transformers.git
2
+ torch