|
import cv2 |
|
import gradio as gr |
|
import tempfile |
|
import torch |
|
import torchaudio |
|
from torchvision.models.detection import fasterrcnn_resnet50_fpn |
|
import torchvision.transforms as transforms |
|
from PIL import Image |
|
import numpy as np |
|
import soundfile as sf |
|
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor |
|
|
|
class FasterRCNNDetector: |
|
def __init__(self): |
|
self.model = fasterrcnn_resnet50_fpn(pretrained=True) |
|
self.model.eval() |
|
self.classes = [ |
|
"__background__", "person", "bicycle", "car", "motorcycle", "airplane", "bus", |
|
"train", "truck", "boat", "traffic light", "fire hydrant", "N/A", "stop sign", |
|
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", |
|
"elephant", "bear", "zebra", "giraffe", "N/A", "backpack", "umbrella", "N/A", "N/A", |
|
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", |
|
"kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", |
|
"bottle", "N/A", "wine glass", "cup", "fork", "knife", "spoon", "bowl", |
|
"banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", |
|
"donut", "cake", "chair", "couch", "potted plant", "bed", "N/A", "dining table", |
|
"N/A", "N/A", "toilet", "N/A", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", |
|
"microwave", "oven", "toaster", "sink", "refrigerator", "N/A", "book", |
|
"clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" |
|
] |
|
|
|
def detect_objects(self, image): |
|
image_pil = Image.fromarray(image) |
|
transform = transforms.Compose([transforms.ToTensor()]) |
|
image_tensor = transform(image_pil).unsqueeze(0) |
|
|
|
with torch.no_grad(): |
|
prediction = self.model(image_tensor) |
|
|
|
boxes = prediction[0]['boxes'] |
|
labels = prediction[0]['labels'] |
|
scores = prediction[0]['scores'] |
|
|
|
for box, label, score in zip(boxes, labels, scores): |
|
box = [int(i) for i in box] |
|
cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2) |
|
cv2.putText(image, self.classes[label], (box[0], box[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (36,255,12), 2) |
|
|
|
return image |
|
|
|
class JarvisModels: |
|
def __init__(self): |
|
self.processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h") |
|
self.model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h") |
|
|
|
async def generate_response(self, prompt): |
|
|
|
generate_kwargs = dict( |
|
temperature=0.6, |
|
max_new_tokens=256, |
|
top_p=0.95, |
|
repetition_penalty=1, |
|
do_sample=True, |
|
seed=42, |
|
) |
|
formatted_prompt = system_instructions1 + prompt + "[JARVIS]" |
|
stream = self.client1.text_generation( |
|
formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True) |
|
output = "" |
|
for response in stream: |
|
output += response.token.text |
|
|
|
return output |
|
|
|
async def transcribe_audio(self, audio_file): |
|
input_audio, _ = torchaudio.load(audio_file) |
|
input_values = self.processor(input_audio, return_tensors="pt").input_values |
|
logits = self.model(input_values).logits |
|
predicted_ids = torch.argmax(logits, dim=-1) |
|
transcription = self.processor.batch_decode(predicted_ids) |
|
return transcription[0] |
|
|
|
detector = FasterRCNNDetector() |
|
|
|
iface = gr.Interface( |
|
fn=[detector.detect_objects, JarvisModels().transcribe_audio], |
|
inputs=gr.inputs.Video(label="Webcam", parameters={"fps": 30}), |
|
outputs=[gr.outputs.Image(), "text"], |
|
title="Vision and Speech Interface", |
|
description="This interface detects objects in the webcam feed and transcribes speech recorded through the microphone." |
|
) |
|
iface.launch() |
|
|
