| import os | |
| import gradio as gr | |
| import torch | |
| import numpy as np | |
| from transformers import pipeline | |
| import torch | |
| print(f"Is CUDA available: {torch.cuda.is_available()}") | |
| print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}") | |
| pipe_flan = pipeline("text2text-generation", model="philschmid/flan-t5-xxl-sharded-fp16", model_kwargs={"load_in_8bit":True, "device_map": "auto"}) | |
| pipe_vanilla = pipeline("text2text-generation", model="t5-large", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16}) | |
| title = "Flan T5 and Vanilla T5" | |
| description = "This demo compares [T5-large](https://huggingface.co/t5-large) and [Flan-T5-XX-large](https://huggingface.co/google/flan-t5-xxl). Note that T5 expects a very specific format of the prompts, so the examples below are not necessarily the best prompts to compare." | |
| def inference(text): | |
| output_flan = pipe_flan(text, max_length=100)[0]["generated_text"] | |
| output_vanilla = pipe_vanilla(text, max_length=100)[0]["generated_text"] | |
| return [output_flan, output_vanilla] | |
| io = gr.Interface( | |
| inference, | |
| gr.Textbox(lines=3), | |
| outputs=[ | |
| gr.Textbox(lines=3, label="Flan T5"), | |
| gr.Textbox(lines=3, label="T5") | |
| ], | |
| title=title, | |
| description=description, | |
| examples=examples | |
| ) | |
| io.launch() |