Spaces:
Running
Running
| import gradio as gr | |
| from transformers import ViltProcessor, ViltForQuestionAnswering, AutoModelForSeq2SeqLM, AutoTokenizer | |
| import torch | |
| import httpcore | |
| setattr(httpcore, 'SyncHTTPTransport', 'AsyncHTTPProxy') | |
| from googletrans import Translator | |
| from googletrans import LANGCODES | |
| import re | |
| # List of acceptable languages | |
| acceptable_languages = set(L.split()[0] for L in LANGCODES) | |
| acceptable_languages.add("mandarin") | |
| acceptable_languages.add("cantonese") | |
| # Translation | |
| def google_translate(question, dest): | |
| translator = Translator() | |
| translation = translator.translate(question, dest=dest) | |
| print("Translation text: " + translation.text) | |
| print("Translation src: " + translation.src) | |
| return (translation.text, translation.src) | |
| # Lang to lang_code mapping | |
| def lang_code_match(accaptable_lang): | |
| # Exception for chinese langs | |
| if accaptable_lang == 'mandarin': | |
| return 'zh-cn' | |
| elif accaptable_lang == 'cantonese' or accaptable_lang == 'chinese': | |
| return 'zh-tw' | |
| # Default | |
| else: | |
| return LANGCODES[accaptable_lang] | |
| # Find destination langauge | |
| def find_dest_language(sentence, src_lang): | |
| pattern = r'\b(' + '|'.join(acceptable_languages) + r')\b' | |
| match = re.search(pattern, sentence, flags=re.IGNORECASE) | |
| if match: | |
| lang_code = lang_code_match(match.group(0).lower()) | |
| print("Destination lang: " + lang_code) | |
| return lang_code | |
| else: | |
| print("Destination lang:" + src_lang) | |
| return src_lang | |
| # Remove destination langauge context | |
| def remove_language_phrase(sentence): | |
| # Bremove "in [acceptable_languages]" or "[acceptable_languages]" and any non-closing punctuation around it | |
| pattern = r'(\b(in\s)?(' + '|'.join(acceptable_languages) + r')\b)[\s,;:.!?]*' | |
| cleaned_sentence = re.sub(pattern, '', sentence, flags=re.IGNORECASE).strip() | |
| print("Language Phrase Removed: " + cleaned_sentence) | |
| return cleaned_sentence | |
| def vqa(image, text): | |
| encoding = vqa_processor(image, text, return_tensors="pt") | |
| with torch.no_grad(): | |
| outputs = vqa_model(**encoding) | |
| logits = outputs.logits | |
| idx = logits.argmax(-1).item() | |
| predicted_answer = vqa_model.config.id2label[idx] | |
| return predicted_answer | |
| def llm(cleaned_sentence, vqa_answer): | |
| # Prepare the input prompt | |
| prompt = ( | |
| f"A question: {cleaned_sentence}\n" | |
| f"An answer: {vqa_answer}.\n" | |
| f"Based on these, answer the question with a complete sentence without extra information." | |
| ) | |
| inputs = flan_tokenizer(prompt, return_tensors="pt") | |
| outputs = flan_model.generate(**inputs, max_length=50) | |
| response = flan_tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| print("T5 prompt: " + prompt) | |
| print("T5 response: " + response) | |
| return response | |
| torch.hub.download_url_to_file('https://media.istockphoto.com/id/1174602891/photo/two-monkeys-mom-and-cub-eat-bananas.jpg?s=612x612&w=0&k=20&c=r7VXi9d1wHhyq3iAk9D2Z3yTZiOJMlLNtjdVRBEjG7g=', 'monkeys.jpg') | |
| torch.hub.download_url_to_file('https://img.freepik.com/premium-photo/man-holds-apple-his-hands_198067-740023.jpg', 'apple.jpg') | |
| vqa_processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa") | |
| vqa_model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa") | |
| flan_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large") | |
| flan_model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-large") | |
| def main(image, text): | |
| en_question, question_src_lang = google_translate(text, dest='en') | |
| dest_lang = find_dest_language(en_question, question_src_lang) | |
| cleaned_sentence = remove_language_phrase(en_question) | |
| vqa_answer = vqa(image, cleaned_sentence) | |
| llm_answer = llm(cleaned_sentence, vqa_answer) | |
| final_answer, _ = google_translate(llm_answer, dest=dest_lang) | |
| print("Final answer: ", final_answer) | |
| return vqa_answer, final_answer | |
| image = gr.Image(type="pil") | |
| question = gr.Textbox(label="Question") | |
| answer = gr.Textbox(label="Predicted answer") | |
| examples = [ | |
| ["monkeys.jpg", "What are the monkeys doing in French?"], | |
| ["apple.jpg", "Qu'est-ce que c'est dans ma main en anglais?"], | |
| ["monkeys.jpg", "In Uyghur, tell me how many monkeys are here?"], | |
| ["apple.jpg", "What color is this fruit? Chinese."] | |
| ] | |
| title = "Cross-lingual VQA" | |
| description = """ | |
| Visual Question Answering (VQA) across Languages | |
| Input an image and ask a question regarding the image. | |
| Click on the examples below to see sample inputs and outputs. | |
| """ | |
| article = """ | |
| Supports questions regarding the following languages: | |
| ['afrikaans', 'albanian', 'amharic', 'arabic', 'armenian', 'azerbaijani', 'basque', 'belarusian', 'bengali', 'bosnian', 'bulgarian', 'catalan', 'cebuano', 'chichewa', 'chinese (simplified)', 'chinese (traditional)', 'corsican', 'croatian', 'czech', 'danish', 'dutch', 'english', 'esperanto', 'estonian', 'filipino', 'finnish', 'french', 'frisian', 'galician', 'georgian', 'german', 'greek', 'gujarati', 'haitian creole', 'hausa', 'hawaiian', 'hebrew', 'hebrew', 'hindi', 'hmong', 'hungarian', 'icelandic', 'igbo', 'indonesian', 'irish', 'italian', 'japanese', 'javanese', 'kannada', 'kazakh', 'khmer', 'korean', 'kurdish (kurmanji)', 'kyrgyz', 'lao', 'latin', 'latvian', 'lithuanian', 'luxembourgish', 'macedonian', 'malagasy', 'malay', 'malayalam', 'maltese', 'maori', 'marathi', 'mongolian', 'myanmar (burmese)', 'nepali', 'norwegian', 'odia', 'pashto', 'persian', 'polish', 'portuguese', 'punjabi', 'romanian', 'russian', 'samoan', 'scots gaelic', 'serbian', 'sesotho', 'shona', 'sindhi', 'sinhala', 'slovak', 'slovenian', 'somali', 'spanish', 'sundanese', 'swahili', 'swedish', 'tajik', 'tamil', 'telugu', 'thai', 'turkish', 'ukrainian', 'urdu', 'uyghur', 'uzbek', 'vietnamese', 'welsh', 'xhosa', 'yiddish', 'yoruba', 'zulu'] | |
| """ | |
| interface = gr.Interface(fn=main, | |
| inputs=[image, question], | |
| outputs=answer, | |
| examples=examples, | |
| title=title, | |
| description=description, | |
| article=article) | |
| interface.launch(debug=True) |