Update app.py

app.py

@@ -1,5 +1,5 @@
 """
-LabOps Log Analyzer Dashboard with CSV file upload, PDF generation, and backend Salesforce integration
+LabOps Log Analyzer Dashboard with CSV file upload, PDF generation, and email alerts
 """
 import gradio as gr
 import pandas as pd
@@ -8,27 +8,16 @@ import logging
 import plotly.express as px
 from sklearn.ensemble import IsolationForest
 from concurrent.futures import ThreadPoolExecutor
-from simple_salesforce import Salesforce
 import os
-import json
 import io
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.mime.application import MIMEApplication
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Salesforce configuration
-try:
-    sf = Salesforce(
-        username='multi-devicelabopsdashboard@sathkrutha.com',
-        password='Team@1234',
-        security_token=os.getenv('SF_SECURITY_TOKEN', ''),
-        domain='login'
-    )
-    logging.info("Salesforce connection established")
-except Exception as e:
-    logging.error(f"Failed to connect to Salesforce: {str(e)}")
-    sf = None
-
 # Try to import reportlab
 try:
     from reportlab.lib.pagesizes import letter
@@ -40,229 +29,6 @@ except ImportError:
     logging.warning("reportlab module not found. PDF generation disabled.")
     reportlab_available = False
 
-# Fetch valid picklist values from Salesforce
-def get_picklist_values(field_name):
-    if sf is None:
-        return []
-    try:
-        obj_desc = sf.SmartLog__c.describe()
-        for field in obj_desc['fields']:
-            if field['name'] == field_name:
-                return [value['value'] for value in field['picklistValues'] if value['active']]
-        return []
-    except Exception as e:
-        logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
-        return []
-
-# Cache picklist values at startup
-status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
-log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
-logging.info(f"Valid Status__c values: {status_values}")
-logging.info(f"Valid Log_Type__c values: {log_type_values}")
-
-# Map invalid picklist values to valid ones
-picklist_mapping = {
-    'Status__c': {
-        'normal': 'Active',
-        'error': 'Inactive',
-        'warning': 'Pending',
-        'ok': 'Active',
-        'failed': 'Inactive'
-    },
-    'Log_Type__c': {
-        'maint': 'Smart Log',
-        'error': 'Cell Analysis',
-        'ops': 'UV Verification',
-        'maintenance': 'Smart Log',
-        'cell': 'Cell Analysis',
-        'uv': 'UV Verification',
-        'weight log': 'Smart Log'
-    }
-}
-
-# Fetch folder ID for "LabOps Reports"
-def get_folder_id(folder_name):
-    if sf is None:
-        return None
-    try:
-        query = f"SELECT Id FROM Folder WHERE Name = '{folder_name}' AND Type = 'Report'"
-        result = sf.query(query)
-        if result['totalSize'] > 0:
-            folder_id = result['records'][0]['Id']
-            logging.info(f"Found folder ID for '{folder_name}': {folder_id}")
-            return folder_id
-        else:
-            logging.error(f"Folder '{folder_name}' not found in Salesforce.")
-            return None
-    except Exception as e:
-        logging.error(f"Failed to fetch folder ID for '{folder_name}': {str(e)}")
-        return None
-
-# Cache the folder ID at startup
-LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
-
-# Salesforce report creation (runs in backend, result not displayed)
-def create_salesforce_reports(df):
-    if sf is None:
-        logging.info("Salesforce connection not available for report creation.")
-        return
-    if not LABOPS_REPORTS_FOLDER_ID:
-        logging.info("Cannot create reports: 'LabOps Reports' folder not found in Salesforce.")
-        return
-
-    try:
-        usage_report_metadata = {
-            "reportMetadata": {
-                "name": f"SmartLog_Usage_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
-                "developerName": f"SmartLog_Usage_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
-                "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
-                "reportFormat": "TABULAR",
-                "reportBooleanFilter": None,
-                "reportFilters": [],
-                "detailColumns": [
-                    "SmartLog__c.Device_Id__c",
-                    "SmartLog__c.Usage_Hours__c"
-                ],
-                "folderId": LABOPS_REPORTS_FOLDER_ID
-            }
-        }
-        logging.info(f"Creating Usage Report with metadata: {json.dumps(usage_report_metadata, indent=2)}")
-        usage_result = sf.restful('analytics/reports', method='POST', json=usage_report_metadata)
-        usage_report_id = usage_result['id']
-        logging.info(f"Usage Report created: {usage_report_id}")
-
-        amc_report_metadata = {
-            "reportMetadata": {
-                "name": f"SmartLog_AMC_Reminders_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
-                "developerName": f"SmartLog_AMC_Reminders_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
-                "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
-                "reportFormat": "TABULAR",
-                "reportBooleanFilter": None,
-                "reportFilters": [],
-                "detailColumns": [
-                    "SmartLog__c.Device_Id__c",
-                    "SmartLog__c.AMC_Date__c"
-                ],
-                "folderId": LABOPS_REPORTS_FOLDER_ID
-            }
-        }
-        logging.info(f"Creating AMC Reminders Report with metadata: {json.dumps(amc_report_metadata, indent=2)}")
-        amc_result = sf.restful('analytics/reports', method='POST', json=amc_report_metadata)
-        amc_report_id = amc_result['id']
-        logging.info(f"AMC Reminders Report created: {amc_report_id}")
-    except Exception as e:
-        logging.error(f"Failed to create Salesforce reports: {str(e)}")
-
-# Save results to Salesforce SmartLog__c (runs in backend, result not displayed)
-def save_to_salesforce(df, reminders_df, summary, anomalies, amc_reminders, insights):
-    if sf is None:
-        logging.info("Salesforce connection not available for saving records.")
-        return
-    
-    reminder_records = []
-    current_date = datetime.now()
-    next_30_days = current_date + timedelta(days=30)
-    
-    if not reminders_df.empty:
-        logging.info(f"Processing {len(reminders_df)} AMC reminder records for saving to Salesforce")
-        for _, row in reminders_df.iterrows():
-            status = str(row['status'])
-            log_type = str(row['log_type'])
-            
-            if status not in status_values:
-                status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
-                if status is None:
-                    logging.warning(f"Skipping reminder record with invalid Status__c: {row['status']}")
-                    continue
-            
-            if log_type not in log_type_values:
-                log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
-                if log_type is None:
-                    logging.warning(f"Skipping reminder record with invalid Log_Type__c: {row['log_type']}")
-                    continue
-
-            amc_date_str = None
-            if pd.notna(row['amc_date']):
-                try:
-                    amc_date = pd.to_datetime(row['amc_date'], errors='coerce')
-                    if pd.isna(amc_date):
-                        logging.error(f"Failed to parse AMC Date for Device ID {row['device_id']}: {row['amc_date']}")
-                    else:
-                        amc_date_str = amc_date.strftime('%Y-%m-%d')
-                        amc_date_dt = datetime.strptime(amc_date_str, '%Y-%m-%d')
-                        if status == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
-                            logging.info(f"AMC Reminder record saved: Device ID {row['device_id']}, AMC Date {amc_date_str}")
-                except Exception as e:
-                    logging.error(f"Failed to parse AMC Date for Device ID {row['device_id']}: {str(e)}")
-                    amc_date_str = None
-
-            record = {
-                'Device_Id__c': str(row['device_id'])[:50],
-                'Log_Type__c': log_type,
-                'Status__c': status,
-                'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
-                'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
-                'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
-                'AMC_Date__c': amc_date_str
-            }
-            reminder_records.append(record)
-    
-    other_records = []
-    reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
-    for _, row in df.iterrows():
-        if row['device_id'] in reminder_device_ids:
-            continue
-
-        status = str(row['status'])
-        log_type = str(row['log_type'])
-        
-        if status not in status_values:
-            status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
-            if status is None:
-                logging.warning(f"Skipping record with invalid Status__c: {row['status']}")
-                continue
-        
-        if log_type not in log_type_values:
-            log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
-            if log_type is None:
-                logging.warning(f"Skipping record with invalid Log_Type__c: {row['log_type']}")
-                continue
-
-        amc_date_str = None
-        if pd.notna(row['amc_date']):
-            try:
-                amc_date = pd.to_datetime(row['amc_date'], errors='coerce')
-                if pd.isna(amc_date):
-                    logging.error(f"Failed to parse AMC Date for Device ID {row['device_id']}: {row['amc_date']}")
-                else:
-                    amc_date_str = amc_date.strftime('%Y-%m-%d')
-                    amc_date_dt = datetime.strptime(amc_date_str, '%Y-%m-%d')
-                    if status == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
-                        logging.info(f"Record qualifies for AMC Reminders: Device ID {row['device_id']}, AMC Date {amc_date_str}")
-            except Exception as e:
-                logging.error(f"Failed to parse AMC Date for Device ID {row['device_id']}: {str(e)}")
-                amc_date_str = None
-
-        record = {
-            'Device_Id__c': str(row['device_id'])[:50],
-            'Log_Type__c': log_type,
-            'Status__c': status,
-            'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
-            'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
-            'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
-            'AMC_Date__c': amc_date_str
-        }
-        other_records.append(record)
-
-    all_records = reminder_records + other_records
-    
-    try:
-        if all_records:
-            sf.bulk.SmartLog__c.insert(all_records)
-            logging.info(f"Saved {len(all_records)} total records to Salesforce (including {len(reminder_records)} AMC reminders)")
-    except Exception as e:
-        logging.error(f"Failed to save to Salesforce: {str(e)}")
-
 # Generate summary and insights without Hugging Face model
 def generate_summary_and_insights(df):
     try:
@@ -325,6 +91,7 @@ def check_amc_reminders(df, current_date):
 def create_usage_chart(agg_data):
     try:
         usage_data = agg_data['usage_per_device']
+        logging.info(f"Usage data for chart: {usage_data.to_string()}")
         if usage_data.empty:
             logging.warning("Usage data is empty.")
             return None
@@ -384,6 +151,7 @@ def create_usage_chart(agg_data):
 def create_downtime_chart(agg_data):
     try:
         downtime_data = agg_data['downtime_per_device']
+        logging.info(f"Downtime data for chart: {downtime_data.to_string()}")
         if downtime_data.empty:
             logging.warning("Downtime data is empty.")
             return None
@@ -439,6 +207,164 @@ def create_downtime_chart(agg_data):
         logging.error(f"Failed to create downtime chart: {str(e)}")
         return None
 
+# Create Daily Log Trends chart
+def create_daily_log_trends_chart(df):
+    try:
+        if df.empty or 'timestamp' not in df.columns:
+            logging.warning("DataFrame is empty or missing 'timestamp' column for Daily Log Trends.")
+            return None
+        
+        # Group by date to count logs per day
+        df['date'] = df['timestamp'].dt.date
+        log_counts = df.groupby('date').size().reset_index(name='log_count')
+        
+        fig = px.line(
+            log_counts,
+            x='date',
+            y='log_count',
+            title="Daily Log Trends",
+            labels={"date": "Date", "log_count": "Number of Logs"}
+        )
+        fig.update_traces(
+            line_color='#4ECDC4',
+            line_width=2
+        )
+        fig.update_layout(
+            title_font=dict(size=18, family="Arial", color="#333333"),
+            font=dict(family="Arial", size=12, color="#333333"),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            margin=dict(l=30, r=30, t=50, b=30),
+            xaxis=dict(
+                title="Date",
+                showgrid=False,
+                title_font=dict(size=14),
+                tickfont=dict(size=12)
+            ),
+            yaxis=dict(
+                title="Number of Logs",
+                gridcolor="#E5E5E5",
+                gridwidth=1,
+                title_font=dict(size=14),
+                tickfont=dict(size=12)
+            )
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create Daily Log Trends chart: {str(e)}")
+        return None
+
+# Create Weekly Uptime Percentage chart
+def create_weekly_uptime_chart(df):
+    try:
+        if df.empty or 'timestamp' not in df.columns or 'downtime' not in df.columns:
+            logging.warning("DataFrame is empty or missing required columns for Weekly Uptime Percentage.")
+            return None
+        
+        # Group by week
+        df['week'] = df['timestamp'].dt.isocalendar().week
+        df['year'] = df['timestamp'].dt.year
+        weekly_data = df.groupby(['year', 'week']).agg({
+            'downtime': 'sum'
+        }).reset_index()
+        
+        # Calculate uptime percentage (assuming 24*7 = 168 hours per week)
+        total_hours_per_week = 168
+        weekly_data['uptime_percentage'] = ((total_hours_per_week - weekly_data['downtime']) / total_hours_per_week) * 100
+        weekly_data['week_label'] = weekly_data.apply(lambda x: f"{x['year']}-W{x['week']:02d}", axis=1)
+        
+        fig = px.bar(
+            weekly_data,
+            x='week_label',
+            y='uptime_percentage',
+            title="Weekly Uptime Percentage",
+            labels={"week_label": "Week", "uptime_percentage": "Uptime Percentage (%)"},
+            color='uptime_percentage',
+            color_continuous_scale=['#FF0000', '#96CEB4']
+        )
+        fig.update_traces(
+            marker_line_color='#333333',
+            marker_line_width=1.5,
+            opacity=0.9
+        )
+        fig.update_layout(
+            title_font=dict(size=18, family="Arial", color="#333333"),
+            font=dict(family="Arial", size=12, color="#333333"),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            margin=dict(l=30, r=30, t=50, b=30),
+            xaxis=dict(
+                title="Week",
+                showgrid=False,
+                tickangle=45,
+                title_font=dict(size=14),
+                tickfont=dict(size=12)
+            ),
+            yaxis=dict(
+                title="Uptime Percentage (%)",
+                gridcolor="#E5E5E5",
+                gridwidth=1,
+                title_font=dict(size=14),
+                tickfont=dict(size=12)
+            ),
+            showlegend=False,
+            bargap=0.2
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create Weekly Uptime Percentage chart: {str(e)}")
+        return None
+
+# Create Anomaly Alerts chart
+def create_anomaly_alerts_chart(df, anomalies_df):
+    try:
+        if df.empty or anomalies_df.empty:
+            logging.warning("DataFrame or anomalies DataFrame is empty for Anomaly Alerts chart.")
+            return None
+        
+        # Prepare data for scatter plot
+        df['is_anomaly'] = df.index.isin(anomalies_df.index)
+        df['color'] = df['is_anomaly'].map({True: 'red', False: 'blue'})
+        
+        fig = px.scatter(
+            df,
+            x='usage_hours',
+            y='downtime',
+            color='color',
+            title="Anomaly Alerts (Red = Anomaly)",
+            labels={"usage_hours": "Usage Hours", "downtime": "Downtime (Hours)"},
+            color_discrete_map={'blue': '#4ECDC4', 'red': '#FF0000'}
+        )
+        fig.update_traces(
+            marker=dict(size=8, line=dict(width=1, color='#333333')),
+            opacity=0.7
+        )
+        fig.update_layout(
+            title_font=dict(size=18, family="Arial", color="#333333"),
+            font=dict(family="Arial", size=12, color="#333333"),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            margin=dict(l=30, r=30, t=50, b=30),
+            xaxis=dict(
+                title="Usage Hours",
+                showgrid=False,
+                title_font=dict(size=14),
+                tickfont=dict(size=12)
+            ),
+            yaxis=dict(
+                title="Downtime (Hours)",
+                gridcolor="#E5E5E5",
+                gridwidth=1,
+                title_font=dict(size=14),
+                tickfont=dict(size=12)
+            ),
+            showlegend=False
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create Anomaly Alerts chart: {str(e)}")
+        return None
+
 # Generate Device Cards HTML
 def generate_device_cards(df):
     try:
@@ -599,6 +525,65 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
         logging.error(f"Failed to generate PDF: {str(e)}", exc_info=True)
         return None
 
+# Send email alert with analysis summary and PDF attachment
+def send_email_alert(summary, anomalies, amc_reminders, pdf_path, recipient_email="recipient@example.com"):
+    try:
+        # Email configuration
+        sender_email = "your_email@gmail.com"  # Replace with your email
+        sender_password = "your_app_password"  # Replace with your app-specific password
+        smtp_server = "smtp.gmail.com"
+        smtp_port = 587
+
+        # Create email message
+        subject = "LabOps Log Analyzer Report - Analysis Completed"
+        body = f"""
+        Dear Recipient,
+
+        The LabOps Log Analyzer has completed its analysis. Below are the key findings:
+
+        **Summary:**
+        {summary}
+
+        **Anomalies Detected:**
+        {anomalies}
+
+        **AMC Reminders:**
+        {amc_reminders}
+
+        The full report is attached as a PDF for your review.
+
+        Regards,
+        LabOps Team
+        """
+        
+        msg = MIMEMultipart()
+        msg['From'] = sender_email
+        msg['To'] = recipient_email
+        msg['Subject'] = subject
+        msg.attach(MIMEText(body, 'plain'))
+
+        # Attach the PDF if it exists
+        if pdf_path and os.path.exists(pdf_path):
+            with open(pdf_path, 'rb') as f:
+                pdf_attachment = MIMEApplication(f.read(), _subtype="pdf")
+                pdf_attachment.add_header(
+                    'Content-Disposition', 'attachment', filename=os.path.basename(pdf_path)
+                )
+                msg.attach(pdf_attachment)
+            logging.info(f"Attached PDF to email: {pdf_path}")
+        else:
+            logging.warning("No PDF file to attach to email.")
+
+        # Send the email
+        with smtplib.SMTP(smtp_server, smtp_port) as server:
+            server.starttls()
+            server.login(sender_email, sender_password)
+            server.sendmail(sender_email, recipient_email, msg.as_string())
+        
+        logging.info(f"Email alert sent to {recipient_email}")
+    except Exception as e:
+        logging.error(f"Failed to send email alert: {str(e)}")
+
 # Main Gradio function
 async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
     try:
@@ -692,12 +677,14 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         }
 
         # Run tasks concurrently
-        with ThreadPoolExecutor(max_workers=2) as executor:
+        with ThreadPoolExecutor(max_workers=3) as executor:
             future_summary_insights = executor.submit(generate_summary_and_insights, filtered_df)
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
             future_usage_chart = executor.submit(create_usage_chart, agg_data)
             future_downtime_chart = executor.submit(create_downtime_chart, agg_data)
+            future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
+            future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
             future_device_cards = executor.submit(generate_device_cards, filtered_df)
 
             summary, insights = future_summary_insights.result()
@@ -709,11 +696,13 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             amc_reminders = f"AMC Reminders\n{amc_reminders}"
             usage_chart = future_usage_chart.result()
             downtime_chart = future_downtime_chart.result()
-            daily_log_chart = None
-            weekly_uptime_chart = None
-            anomaly_alerts_chart = None
+            daily_log_chart = future_daily_log_chart.result()
+            weekly_uptime_chart = future_weekly_uptime_chart.result()
             device_cards = future_device_cards.result()
 
+        # Generate Anomaly Alerts chart after anomalies are detected
+        anomaly_alerts_chart = create_anomaly_alerts_chart(filtered_df, anomalies_df)
+
         # Generate the log preview as an HTML table
         preview_html = """
         <style>
@@ -802,6 +791,9 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         else:
             status_msg = "Analysis completed, but some data is missing for PDF generation."
 
+        # Send email alert
+        send_email_alert(summary, anomalies, amc_reminders, pdf_file)
+
         elapsed_time = (datetime.now() - start_time).total_seconds()
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
         if elapsed_time > 10:
@@ -868,7 +860,7 @@ try:
         .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
     """) as iface:
         gr.Markdown("<h1>LabOps Log Analyzer Dashboard</h1>")
-        gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data. A PDF report will be generated automatically.")
+        gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data. A PDF report will be generated automatically, and an email alert will be sent.")
 
         last_modified_state = gr.State(value=None)
         summary_state = gr.State()