AkashKhatri commited on
Commit
ba534be
1 Parent(s): b82b792
Files changed (2) hide show
  1. sign_asl_cnn_30_epochs.h5 +3 -0
  2. streamlit_app.py +152 -0
sign_asl_cnn_30_epochs.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bb7dbc904898a1b2e60dc5d60fc045bb5dea4fd422fb292b3de22f83e49a2d9
3
+ size 495899136
streamlit_app.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import numpy as np
3
+ from keras.models import load_model
4
+ import cv2
5
+ from io import BytesIO
6
+ import mediapipe as mp
7
+
8
+ # Load the model
9
+ model = load_model('sign_asl_cnn_30_epochs.h5')
10
+ class_labels = {i: str(i) if i < 10 else chr(65 + i - 10) for i in range(36)}
11
+
12
+ # Function to preprocess the image
13
+ def preprocess_image(image):
14
+ image = cv2.resize(image, (200, 200))
15
+ image = image / 255.0
16
+ image = image.reshape(1, 200, 200, 3)
17
+ return image
18
+
19
+ # Function to predict the sign language letter
20
+ def predict_letter(image):
21
+ processed_image = preprocess_image(image)
22
+ predictions = model.predict(processed_image)
23
+ predicted_class = np.argmax(predictions, axis=1)[0]
24
+ sign_letter = class_labels[predicted_class]
25
+ return sign_letter
26
+
27
+ # Function to detect hands in the image
28
+ def detect_hands(image):
29
+ mp_hands = mp.solutions.hands
30
+ hands = mp_hands.Hands()
31
+ margin = 15
32
+
33
+ # Convert the image to RGB
34
+ image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
35
+
36
+ # Process the image and get the hand landmarks
37
+ results = hands.process(image_rgb)
38
+
39
+ if results.multi_hand_landmarks:
40
+ for landmarks in results.multi_hand_landmarks:
41
+ # Get bounding box coordinates of the hand
42
+ landmarks_xy = [(int(landmark.x * image.shape[1]), int(landmark.y * image.shape[0]))
43
+ for landmark in landmarks.landmark]
44
+
45
+ # Define the bounding box for the hand
46
+ x_min = max(0, min(landmarks_xy, key=lambda x: x[0])[0] - margin)
47
+ y_min = max(0, min(landmarks_xy, key=lambda x: x[1])[1] - margin)
48
+ x_max = min(image.shape[1], max(landmarks_xy, key=lambda x: x[0])[0] + margin)
49
+ y_max = min(image.shape[0], max(landmarks_xy, key=lambda x: x[1])[1] + margin)
50
+
51
+ # Extract the hand region
52
+ roi = image[y_min:y_max, x_min:x_max]
53
+
54
+ # Check if the ROI is empty
55
+ if roi.size == 0:
56
+ continue
57
+
58
+ # Resize the ROI to match your model's input shape
59
+ roi = cv2.resize(roi, (200, 200), interpolation=cv2.INTER_AREA)
60
+ hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)
61
+
62
+ lower_yellow = np.array([93, 72, 51])
63
+ upper_yellow = np.array([224, 194, 183])
64
+ mask = cv2.inRange(hsv, lower_yellow, upper_yellow)
65
+ roi = cv2.bitwise_and(roi, roi, mask=mask)
66
+ roi = roi.reshape(1, 200, 200, 3) # Ensure it matches your model's input shape
67
+
68
+ # Make predictions using your classifier
69
+ predictions = model.predict(roi)
70
+ predicted_class = int(np.argmax(predictions, axis=1)[0])
71
+ result = class_labels[predicted_class]
72
+
73
+ # Draw result on the image
74
+ cv2.putText(image, str(result), (x_min, y_min - 10),
75
+ cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2)
76
+
77
+ # Draw bounding box on the image
78
+ cv2.rectangle(image, (x_min, y_min), (x_max, y_max), (255, 0, 0), 2)
79
+
80
+ return image
81
+
82
+ # Streamlit app
83
+ st.title('Sign Language Recognition')
84
+
85
+ # Sidebar with radio button for Upload/Webcam
86
+ selected_option = st.sidebar.radio("Select Option", ["Upload", "Webcam"], index=0)
87
+
88
+ if selected_option == "Upload":
89
+ uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "png"])
90
+
91
+ if uploaded_file is not None:
92
+ if st.button('Predict'):
93
+ contents = uploaded_file.read()
94
+ nparr = np.frombuffer(contents, np.uint8)
95
+ image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
96
+
97
+ # Make the prediction
98
+ predicted_letter = predict_letter(image)
99
+
100
+ # Display the predicted letter
101
+ st.write('Predicted Letter:', predicted_letter)
102
+
103
+ elif selected_option == "Webcam":
104
+ # Placeholder for webcam frame
105
+ webcam_frame = st.empty()
106
+
107
+ # Placeholder for predicted letter in webcam mode
108
+ predicted_letter_webcam = st.empty()
109
+
110
+ # Placeholder for webcam capture status
111
+ webcam_capture_status = st.empty()
112
+
113
+ # Placeholder for webcam stop button
114
+ webcam_stop_button = st.empty()
115
+
116
+ # Placeholder for webcam status
117
+ webcam_status = st.empty()
118
+
119
+ # Placeholder for webcam button
120
+ webcam_button = st.button("Start Webcam")
121
+
122
+ if webcam_button:
123
+ webcam_status.text("Webcam is on.")
124
+ webcam_stop_button = st.button("Stop Webcam")
125
+
126
+ # OpenCV video capture
127
+ cap = cv2.VideoCapture(0)
128
+
129
+ while True:
130
+ # Read the frame from the webcam
131
+ ret, frame = cap.read()
132
+
133
+ # Display the frame in Streamlit
134
+ webcam_frame.image(frame, channels="BGR")
135
+
136
+ # Detect hands in the current frame
137
+ frame = detect_hands(frame)
138
+
139
+ # Convert the frame to JPEG format
140
+ _, jpeg = cv2.imencode(".jpg", frame)
141
+
142
+ # Display the predicted letter
143
+ predicted_letter = predict_letter(frame)
144
+ predicted_letter_webcam.text(f"Predicted Letter: {predicted_letter}")
145
+
146
+ # Check if the "Stop Webcam" button is clicked
147
+ if webcam_stop_button:
148
+ webcam_status.text("Webcam is off.")
149
+ break
150
+
151
+ # Release the webcam when done
152
+ cap.release()