Spaces:
Sleeping
Sleeping
File size: 2,478 Bytes
9df97a9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import cvzone
import cv2
from cvzone.HandTrackingModule import HandDetector
import numpy as np
import google.generativeai as genai
from PIL import Image
import streamlit as st
st.set_page_config(layout="wide")
st.image('img.png')
col1, col2 = st.columns([3,2])
with col1:
run = st.checkbox('Run', value=True)
FRAME_WINDOW = st.image([])
with col2:
st.title("Answer")
output_text_area = st.subheader("")
genai.configure(api_key="AIzaSyAu7w2tMO4kIAiB-RDMh8vywmF8OqBjpQk")
model = genai.GenerativeModel('gemini-1.5-flash')
# Initialize the webcam to capture video
cap = cv2.VideoCapture(0) # Try using 0 for built-in camera
cap.set(3,1280)
cap.set(4,720)
detector = HandDetector(staticMode=False, maxHands=1, modelComplexity=1, detectionCon=0.7, minTrackCon=0.5)
def getHandInfo(img):
if img is None or not img.any():
return None
hands, img = detector.findHands(img, draw=False, flipType=True)
if hands:
hand = hands[0]
lmList = hand["lmList"]
fingers = detector.fingersUp(hand)
return fingers, lmList
else:
return None
def draw(info, prev_pos, canvas):
fingers, lmList = info
current_pos = None
if fingers == [0, 1, 0, 0, 0]:
current_pos = lmList[8][0:2]
if prev_pos is None: prev_pos = current_pos
cv2.line(canvas, current_pos, prev_pos, (255, 0, 255), 10)
elif fingers == [1, 0, 0, 0, 0]:
canvas = np.zeros_like(img)
return current_pos, canvas
def sendToAI(model, canvas, fingers):
if fingers == [1, 1, 1, 1, 0]:
pil_image = Image.fromarray(canvas)
response = model.generate_content(["Solve this math problem", pil_image])
return response.text
prev_pos = None
canvas = None
output_text = ""
while True:
success, img = cap.read()
if not success or img is None:
continue # Skip this iteration if the frame is not captured properly
img = cv2.flip(img, 1)
if canvas is None:
canvas = np.zeros_like(img)
info = getHandInfo(img)
if info:
fingers, lmList = info
prev_pos, canvas = draw(info, prev_pos, canvas)
output_text = sendToAI(model, canvas, fingers)
image_combined = cv2.addWeighted(img, 0.7, canvas, 0.3, 0)
FRAME_WINDOW.image(image_combined, channels="BGR")
if output_text:
output_text_area.text(output_text)
cv2.waitKey(1)
|