Spaces:
Runtime error
Runtime error
| from transformers import SegformerFeatureExtractor, SegformerForSemanticSegmentation | |
| from transformers import pipeline | |
| from PIL import Image | |
| import gradio as gr | |
| import torch | |
| #this converts text to speech | |
| fastspeech = gr.Interface.load("huggingface/facebook/fastspeech2-en-ljspeech") | |
| #this function detects the objects in the room | |
| def object_classify(img1,img2): | |
| feature_extractor = SegformerFeatureExtractor.from_pretrained("nvidia/segformer-b2-finetuned-ade-512-512") | |
| model = SegformerForSemanticSegmentation.from_pretrained("nvidia/segformer-b2-finetuned-ade-512-512") | |
| object_detector = pipeline(task="image-segmentation", model = model, feature_extractor = feature_extractor) | |
| #list of dictionaries | |
| dict_obj1 = object_detector(img1) | |
| dict_obj2 = object_detector(img2) | |
| #list of object labels present in the image | |
| objects_1=[] | |
| objects_2=[] | |
| #this is will read by the fastspeech | |
| tts_words=['The missing items are'] | |
| #gets the label from each dictionary | |
| for i in dict_obj1: | |
| objects_1.append(i['label']) | |
| for j in dict_obj2: | |
| objects_2.append(j['label']) | |
| #gets the uncommon elements from the 2 lists | |
| missing_objects= list(set(objects_1)-set(objects_2)) | |
| if len(missing_objects)==0: | |
| tts_words.append('None') | |
| elif len(missing_objects)==1: | |
| tts_words[0]='The missing item is a' | |
| tts_words.extend(missing_objects) | |
| else: | |
| tts_words.extend(missing_objects) | |
| gonna_process=' '.join(tts_words) | |
| return missing_objects, fastspeech(gonna_process) | |
| TITLE = 'Missing Items using Nvidia Segformer' | |
| DESCRIPTION = 'Input two pictures. First image being the original and second is the one with the missing item/s. This will output a list of items that are missing and an audio version of it' | |
| EXAMPLES = [['Bedroom_1.jpg'],['Bedroom_2.jpg']] | |
| INPUTS=[gr.inputs.Image(type = 'pil', label='Original Image'),gr.inputs.Image(type = 'pil', label='Second Image')] | |
| OUTPUTS=[gr.outputs.Textbox(label='Missing Item/s is/are'),gr.outputs.Audio(type="auto", label="Missing Items Audio")] | |
| interface=gr.Interface(object_classify, | |
| INPUTS, | |
| OUTPUTS, | |
| examples = EXAMPLES, | |
| title = TITLE, | |
| description=DESCRIPTION, allow_flagging="never") | |
| interface.launch() |