Update app.py

app.py

@@ -18,7 +18,11 @@ working_alphabets = ''.join(set(all_alphabets) - set(excluded_alphabets))
 # Function to load the Random Forest model
 @st.cache_resource
 def load_model():
-    """Load the Random Forest model from a file."""
+    """Load the pre-trained Random Forest model.
+
+    Returns:
+        model: The loaded Random Forest model.
+    """
     try:
         return joblib.load('best_random_forest_model.pkl')
     except Exception as e:
@@ -34,13 +38,13 @@ if model is None:
 
 # Function to normalize landmarks
 def normalize_landmarks(landmarks):
-    """Normalize the hand landmarks by centering and scaling.
+    """Normalize the landmark coordinates.
 
     Args:
-        landmarks (np.ndarray): Array of hand landmarks.
+        landmarks (np.ndarray): The array of landmark coordinates.
 
     Returns:
-        np.ndarray: Normalized landmarks.
+        np.ndarray: Normalized landmark coordinates.
     """
     center = np.mean(landmarks, axis=0)
     landmarks_centered = landmarks - center
@@ -53,10 +57,10 @@ def calculate_angles(landmarks):
     """Calculate angles between hand landmarks.
 
     Args:
-        landmarks (np.ndarray): Array of normalized hand landmarks.
+        landmarks (np.ndarray): The array of normalized landmark coordinates.
 
     Returns:
-        list: List of calculated angles.
+        list: List of angles between landmarks.
     """
     angles = []
     for i in range(20):
@@ -69,15 +73,13 @@ def calculate_angles(landmarks):
 
 # Function to process image and predict alphabet
 def process_and_predict(image):
-    """Process the input image and predict the ASL sign.
+    """Process the uploaded image to predict the ASL sign.
 
     Args:
-        image (np.ndarray): The input image containing a hand sign.
+        image (np.ndarray): The uploaded image in BGR format.
 
     Returns:
-        tuple: (probabilities, landmarks) where probabilities are 
-               the predicted probabilities for each class and landmarks 
-               are the detected hand landmarks.
+        tuple: (probabilities of each class, detected landmarks)
     """
     mp_hands = mp.solutions.hands
     with mp_hands.Hands(static_image_mode=True, max_num_hands=1, min_detection_confidence=0.5) as hands:
@@ -95,3 +97,119 @@ def process_and_predict(image):
             probabilities = model.predict_proba(angles_df)[0]
             return probabilities, landmarks
     
+    return None, None
+
+# Function to plot hand landmarks
+def plot_hand_landmarks(landmarks, title):
+    """Plot the detected hand landmarks.
+
+    Args:
+        landmarks (np.ndarray): The array of landmark coordinates.
+        title (str): The title for the plot.
+
+    Returns:
+        Figure: The matplotlib figure object with plotted landmarks.
+    """
+    fig, ax = plt.subplots(figsize=(5, 5))
+    ax.scatter(landmarks[:, 0], landmarks[:, 1], c='blue', s=20)
+    mp_hands = mp.solutions.hands
+    for connection in mp_hands.HAND_CONNECTIONS:
+        start_idx = connection[0]
+        end_idx = connection[1]
+        ax.plot([landmarks[start_idx, 0], landmarks[end_idx, 0]],
+                [landmarks[start_idx, 1], landmarks[end_idx, 1]], 'r-', linewidth=1)
+    ax.invert_yaxis()
+    ax.set_title(title, fontsize=12)
+    ax.axis('off')
+    return fig
+
+# README content
+readme_content = f"""
+## How it works
+
+This ASL Recognition App uses image processing and machine learning to recognize American Sign Language (ASL) hand signs.
+
+1. **Image Upload**: Users can upload an image of an ASL hand sign.
+2. **Hand Detection**: The app uses MediaPipe to detect hand landmarks in the image.
+3. **Feature Extraction**: Angles between hand landmarks are calculated and normalized.
+4. **Prediction**: A Random Forest model predicts the ASL sign based on the extracted features.
+5. **Visualization**: The app displays the detected hand landmarks and top predictions.
+
+### Supported Alphabets
+
+The app currently works for the following ASL alphabets:
+{', '.join(working_alphabets)}
+
+The app does not support or may not work correctly for:
+{', '.join(excluded_alphabets)}
+
+Note: The model's performance may vary and is subject to improvement.
+
+The "View Hand Landmarks" tab allows users to see hand landmarks for pre-loaded ASL signs.
+"""
+
+# Streamlit app
+st.title("ASL Recognition App")
+
+# Display README content
+st.sidebar.markdown(readme_content)
+
+# Create tabs for different functionalities
+tab1, tab2 = st.tabs(["Predict ASL Sign", "View Hand Landmarks"])
+
+with tab1:
+    st.header("Predict ASL Sign")
+    uploaded_file = st.file_uploader("Upload an image of an ASL sign", type=["jpg", "jpeg", "png"])
+
+    if uploaded_file is not None:
+        try:
+            image = cv2.imdecode(np.frombuffer(uploaded_file.read(), np.uint8), 1)
+            if image is not None:
+                col1, col2 = st.columns(2)
+                with col1:
+                    st.image(image, caption="Uploaded Image", use_column_width=True)
+                
+                probabilities, landmarks = process_and_predict(image)
+                
+                if probabilities is not None and landmarks is not None:
+                    with col2:
+                        st.subheader("Top 5 Predictions:")
+                        top_indices = np.argsort(probabilities)[::-1][:5]
+                        for i in top_indices:
+                            st.write(f"{model.classes_[i]}: {probabilities[i]:.2f}")
+                    
+                    fig = plot_hand_landmarks(landmarks, "Detected Hand Landmarks")
+                    st.pyplot(fig)
+                else:
+                    st.write("No hand detected in the image.")
+            else:
+                st.error("Failed to load the image. The file might be corrupted.")
+        except Exception as e:
+            st.error(f"An error occurred while processing the image: {str(e)}")
+
+with tab2:
+    st.header("View Hand Landmarks")
+    
+    selected_alphabets = st.multiselect("Select alphabets to view landmarks:", list(working_alphabets))
+
+    if selected_alphabets:
+        cols = st.columns(4)  # 4 columns for smaller images
+        for idx, alphabet in enumerate(selected_alphabets):
+            with cols[idx % 4]:
+                image_path = os.path.join('asl test set', f'{alphabet.lower()}.jpeg')
+                if os.path.exists(image_path):
+                    try:
+                        image = cv2.imread(image_path)
+                        if image is not None:
+                            probabilities, landmarks = process_and_predict(image)
+                            if landmarks is not None:
+                                fig = plot_hand_landmarks(landmarks, f"Hand Landmarks for {alphabet}")
+                                st.pyplot(fig)
+                            else:
+                                st.error(f"No hand detected for {alphabet}")
+                        else:
+                            st.error(f"Failed to load image for {alphabet}")
+                    except Exception as e:
+                        st.error(f"Error processing image for {alphabet}")
+                else:
+                    st.error(f"Image not found for {alphabet}")