Spaces:
Runtime error
Runtime error
| ##### VQA MED Demo | |
| import gradio as gr | |
| from transformers import ViltProcessor, ViltForQuestionAnswering | |
| import torch | |
| torch.hub.download_url_to_file('http://images.cocodataset.org/val2017/000000039769.jpg', 'cats.jpg') | |
| processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa") | |
| model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa") | |
| def answer_question(image, text): | |
| encoding = processor(image, text, return_tensors="pt") | |
| # forward pass | |
| with torch.no_grad(): | |
| outputs = model(**encoding) | |
| logits = outputs.logits | |
| idx = logits.argmax(-1).item() | |
| predicted_answer = model.config.id2label[idx] | |
| return predicted_answer | |
| image = gr.Image(type="pil") | |
| question = gr.Textbox(label="Question") | |
| answer = gr.Textbox(label="Predicted answer") | |
| examples = [["cats.jpg", "How many cats are there?"]] | |
| title = "Interactive Visual Question Answering demo (BigMed@ai: Artificial Intelligence for Large-Scale Medical Image Analysis)" | |
| description = "<div style='display: flex;align-items: center;justify-content: space-between;'><p style='width:60vw;'>Gradio Demo for VQA medical model trained on PathVQA dataset, To use it, upload your image and type a question and click 'submit', or click one of the examples to load them.</p><a href='https://github.com/dandelin/ViLT' target='_blank' class='link'><img src='file/GitHub.png' style='justify-self:margin-top:0.5em;center; width:calc(200px + 5vw);'></a></div>" | |
| ### link to paper and github code | |
| article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2102.03334' target='_blank'>BigMed@ai</a> | <a href='https://github.com/dandelin/ViLT' target='_blank'>Github Repo</a></p>" | |
| interface = gr.Interface(fn=answer_question, | |
| inputs=[image, question], | |
| outputs=answer, | |
| examples=examples, | |
| title=title, | |
| description=description, | |
| article=article, | |
| ) | |
| interface.launch(debug=True, enable_queue=True) | |