Spaces:

nielsr
/

comparing-captioning-models

Running

App Files Files Community

comparing-captioning-models / app.py

hysts HF staff

Some updates (#53)

4d2a097 verified 4 days ago

raw history blame contribute delete

No virus

5.26 kB

	from __future__ import annotations

	import gradio as gr
	import torch
	from gradio_client import Client, file

	DESCRIPTION = "# Comparing image captioning models"
	ORIGINAL_SPACE_INFO = """\
	- [GIT-large fine-tuned on COCO](https://huggingface.co/spaces/hysts/image-captioning-with-git)
	- [BLIP-large](https://huggingface.co/spaces/hysts/image-captioning-with-blip)
	- [BLIP-2 OPT 6.7B](https://huggingface.co/spaces/merve/BLIP2-with-transformers)
	- [BLIP-2 T5-XXL](https://huggingface.co/spaces/hysts/BLIP2)
	- [InstructBLIP](https://huggingface.co/spaces/hysts/InstructBLIP)
	- [Fuyu-8B](https://huggingface.co/spaces/adept/fuyu-8b-demo)
	"""

	torch.hub.download_url_to_file("http://images.cocodataset.org/val2017/000000039769.jpg", "cats.jpg")
	torch.hub.download_url_to_file(
	"https://huggingface.co/datasets/nielsr/textcaps-sample/resolve/main/stop_sign.png", "stop_sign.png"
	)
	torch.hub.download_url_to_file(
	"https://cdn.openai.com/dall-e-2/demos/text2im/astronaut/horse/photo/0.jpg", "astronaut.jpg"
	)


	def generate_caption_git(image_path: str) -> str:
	try:
	client = Client("hysts/image-captioning-with-git")
	return client.predict(file(image_path), api_name="/caption")
	except Exception:
	gr.Warning("The GIT-large Space is currently unavailable. Please try again later.")
	return ""


	def generate_caption_blip(image_path: str) -> str:
	try:
	client = Client("hysts/image-captioning-with-blip")
	return client.predict(file(image_path), "A picture of", api_name="/caption")
	except Exception:
	gr.Warning("The BLIP-large Space is currently unavailable. Please try again later.")
	return ""


	def generate_caption_blip2_opt(image_path: str) -> str:
	try:
	client = Client("merve/BLIP2-with-transformers")
	return client.predict(
	file(image_path),
	"Beam search",
	1, # temperature
	1, # length penalty
	1.5, # repetition penalty
	api_name="/caption",
	)
	except Exception:
	gr.Warning("The BLIP2 OPT6.7B Space is currently unavailable. Please try again later.")
	return ""


	def generate_caption_blip2_t5xxl(image_path: str) -> str:
	try:
	client = Client("hysts/BLIP2")
	return client.predict(
	file(image_path),
	"Beam search",
	1, # temperature
	1, # length penalty
	1.5, # repetition penalty
	50, # max length
	1, # min length
	5, # number of beams
	0.9, # top p
	api_name="/caption",
	)
	except Exception:
	gr.Warning("The BLIP2 T5-XXL Space is currently unavailable. Please try again later.")
	return ""


	def generate_caption_instructblip(image_path: str) -> str:
	try:
	client = Client("hysts/InstructBLIP")
	return client.predict(
	file(image_path),
	"Describe the image.",
	"Beam search",
	5, # beam size
	256, # max length
	1, # min length
	0.9, # top p
	1.5, # repetition penalty
	1.0, # length penalty
	1.0, # temperature
	api_name="/run",
	)
	except Exception:
	gr.Warning("The InstructBLIP Space is currently unavailable. Please try again later.")
	return ""


	def generate_caption_fuyu(image_path: str) -> str:
	try:
	client = Client("adept/fuyu-8b-demo")
	return client.predict(
	image_path,
	"Generate a coco style caption.\n",
	fn_index=3,
	)
	except Exception:
	gr.Warning("The Fuyu-8B Space is currently unavailable. Please try again later.")
	return ""


	with gr.Blocks(css="style.css") as demo:
	gr.Markdown(DESCRIPTION)
	with gr.Row():
	with gr.Column():
	input_image = gr.Image(type="filepath")
	run_button = gr.Button("Caption")
	with gr.Column():
	out_git = gr.Textbox(label="GIT-large fine-tuned on COCO")
	out_blip = gr.Textbox(label="BLIP-large")
	out_blip2_opt = gr.Textbox(label="BLIP-2 OPT 6.7B")
	out_blip2_t5xxl = gr.Textbox(label="BLIP-2 T5-XXL")
	out_instructblip = gr.Textbox(label="InstructBLIP")
	out_fuyu = gr.Textbox(label="Fuyu-8B")

	gr.Examples(
	examples=[
	"cats.jpg",
	"stop_sign.png",
	"astronaut.jpg",
	],
	inputs=input_image,
	)

	with gr.Accordion(label="The original Spaces can be found here:", open=False):
	gr.Markdown(ORIGINAL_SPACE_INFO)

	fn_out_pairs = [
	(generate_caption_git, out_git),
	(generate_caption_blip, out_blip),
	(generate_caption_blip2_opt, out_blip2_opt),
	(generate_caption_blip2_t5xxl, out_blip2_t5xxl),
	(generate_caption_instructblip, out_instructblip),
	(generate_caption_fuyu, out_fuyu),
	]
	for fn, out in fn_out_pairs:
	run_button.click(
	fn=fn,
	inputs=input_image,
	outputs=out,
	api_name=False,
	)

	if __name__ == "__main__":
	demo.queue(max_size=20, api_open=False).launch(show_api=False)