Update app.py

app.py

@@ -14,6 +14,7 @@ from simple_salesforce import Salesforce
 import os
 import json
 import io
+import time
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -367,34 +368,82 @@ def generate_dashboard_insights(df):
         logging.error(f"Dashboard insights generation failed: {str(e)}")
         return f"Dashboard insights generation failed: {str(e)}"
 
-# Create usage chart (existing chart for usage hours per device)
+# Create usage chart with visual indicators for spikes
 def create_usage_chart(df):
     try:
         usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
         if len(usage_data) > 5:
             usage_data = usage_data.nlargest(5, "usage_hours")
-        custom_colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4']
+        
+        # Define a threshold for usage spikes (e.g., 75th percentile + 1.5 * IQR)
+        q75, q25 = usage_data["usage_hours"].quantile([0.75, 0.25])
+        iqr = q75 - q25
+        spike_threshold = q75 + 1.5 * iqr
+        usage_data["color"] = usage_data["usage_hours"].apply(
+            lambda x: "red" if x > spike_threshold else "blue"
+        )
+
+        custom_colors = usage_data["color"].tolist()
         fig = px.bar(
             usage_data,
             x="device_id",
             y="usage_hours",
-            title="Usage Hours per Device",
+            title="Usage Hours per Device (Red = Usage Spike)",
             labels={"device_id": "Device ID", "usage_hours": "Usage Hours"},
-            color="device_id",
-            color_discrete_sequence=custom_colors
+            color="color",
+            color_discrete_map={"blue": "#4ECDC4", "red": "#FF0000"}
         )
         fig.update_layout(
             title_font_size=16,
             margin=dict(l=20, r=20, t=40, b=20),
             plot_bgcolor="white",
             paper_bgcolor="white",
-            font=dict(size=12)
+            font=dict(size=12),
+            showlegend=False
         )
         return fig
     except Exception as e:
         logging.error(f"Failed to create usage chart: {str(e)}")
         return None
 
+# Create downtime chart with visual indicators for spikes
+def create_downtime_chart(df):
+    try:
+        downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
+        if len(downtime_data) > 5:
+            downtime_data = downtime_data.nlargest(5, "downtime")
+        
+        # Define a threshold for downtime spikes (e.g., 75th percentile + 1.5 * IQR)
+        q75, q25 = downtime_data["downtime"].quantile([0.75, 0.25])
+        iqr = q75 - q25
+        spike_threshold = q75 + 1.5 * iqr
+        downtime_data["color"] = downtime_data["downtime"].apply(
+            lambda x: "red" if x > spike_threshold else "purple"
+        )
+
+        custom_colors = downtime_data["color"].tolist()
+        fig = px.bar(
+            downtime_data,
+            x="device_id",
+            y="downtime",
+            title="Downtime per Device (Red = Downtime Spike)",
+            labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"},
+            color="color",
+            color_discrete_map={"purple": "#800080", "red": "#FF0000"}
+        )
+        fig.update_layout(
+            title_font_size=16,
+            margin=dict(l=20, r=20, t=40, b=20),
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            font=dict(size=12),
+            showlegend=False
+        )
+        return fig
+    except Exception as e:
+        logging.error(f"Failed to create downtime chart: {str(e)}")
+        return None
+
 # Create daily log trends chart
 def create_daily_log_trends_chart(df):
     try:
@@ -557,12 +606,34 @@ def generate_device_cards(df):
         logging.error(f"Failed to generate device cards: {str(e)}", exc_info=True)
         return f'<p>Error generating device cards: {str(e)}</p>'
 
-# Generate PDF content (updated to include new dashboard data)
-def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart):
+# Generate monthly status summary for PDF
+def generate_monthly_status(df, selected_month):
+    try:
+        total_devices = df['device_id'].nunique()
+        total_usage_hours = df['usage_hours'].sum()
+        total_downtime = df['downtime'].sum()
+        avg_usage_per_device = total_usage_hours / total_devices if total_devices > 0 else 0
+        avg_downtime_per_device = total_downtime / total_devices if total_devices > 0 else 0
+
+        summary = f"""
+        Monthly Status for {selected_month}:
+        - Total Devices: {total_devices}
+        - Total Usage Hours: {total_usage_hours:.2f}
+        - Total Downtime Hours: {total_downtime:.2f}
+        - Average Usage per Device: {avg_usage_per_device:.2f} hours
+        - Average Downtime per Device: {avg_downtime_per_device:.2f} hours
+        """
+        return summary
+    except Exception as e:
+        logging.error(f"Failed to generate monthly status: {str(e)}")
+        return f"Failed to generate monthly status: {str(e)}"
+
+# Generate PDF content with monthly status
+def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
     if not reportlab_available:
         return None
     try:
-        pdf_path = f"analysis_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
+        pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
         doc = SimpleDocTemplate(pdf_path, pagesize=letter)
         styles = getSampleStyleSheet()
         story = []
@@ -570,10 +641,17 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
         def safe_paragraph(text, style):
             return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
 
-        story.append(Paragraph("LabOps Log Analysis Report", styles['Title']))
+        story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
         story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
         story.append(Spacer(1, 12))
 
+        # Add monthly status summary if a month is selected
+        if selected_month != "All":
+            monthly_status = generate_monthly_status(df, selected_month)
+            story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
+            story.append(safe_paragraph(monthly_status, styles['Normal']))
+            story.append(Spacer(1, 12))
+
         story.append(Paragraph("Summary Report", styles['Heading2']))
         story.append(safe_paragraph(summary or "No summary available.", styles['Normal']))
         story.append(Spacer(1, 12))
@@ -611,6 +689,10 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
 
         story.append(Paragraph("Anomaly Alerts Chart", styles['Heading2']))
         story.append(Paragraph("[Chart placeholder - see dashboard for Anomaly Alerts]", styles['Normal']))
+        story.append(Spacer(1, 12))
+
+        story.append(Paragraph("Downtime Chart", styles['Heading2']))
+        story.append(Paragraph("[Chart placeholder - see dashboard for Downtime per Device]", styles['Normal']))
 
         doc.build(story)
         logging.info(f"PDF generated at {pdf_path}")
@@ -620,17 +702,25 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
         return None
 
 # Main Gradio function with optimized performance
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
     try:
         start_time = datetime.now()
+
+        # Check if file exists and get its last modified time
         if not file_obj:
-            return "No file uploaded.", "No data to preview.", None, '<p>No device cards available.</p>', None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None
+            return "No file uploaded.", "No data to preview.", None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
+        
+        file_path = file_obj.name
+        current_modified_time = os.path.getmtime(file_path)
+        
+        # Skip processing if the file hasn't changed
+        if last_modified_state and current_modified_time == last_modified_state:
+            return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state
 
-        file_name = file_obj.name
-        logging.info(f"Processing file: {file_name}")
+        logging.info(f"Processing file: {file_path}, last modified: {current_modified_time}")
         
-        if not file_name.endswith(".csv"):
-            return "Please upload a CSV file.", "", None, '<p>No device cards available.</p>', None, None, None, "", "", "", None
+        if not file_path.endswith(".csv"):
+            return "Please upload a CSV file.", "", None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
 
         required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
         dtypes = {
@@ -641,16 +731,16 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             "downtime": "float32",
             "amc_date": "string"
         }
-        df = pd.read_csv(file_obj, dtype=dtypes)
+        df = pd.read_csv(file_path, dtype=dtypes)
         missing_columns = [col for col in required_columns if col not in df.columns]
         if missing_columns:
-            return f"Missing columns: {missing_columns}", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None
+            return f"Missing columns: {missing_columns}", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
         
         # Convert timestamp and amc_date to datetime
         df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
         df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         if df.empty:
-            return "No data available.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None
+            return "No data available.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
         # Log DataFrame before filtering
         logging.info(f"DataFrame before filtering:\n{df.head().to_string()}")
@@ -680,10 +770,19 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
             logging.info(f"After date range filter: {filtered_df.shape[0]} rows")
 
+        # Month filter
+        if month_filter and month_filter != "All":
+            selected_date = pd.to_datetime(month_filter, format="%B %Y")
+            filtered_df = filtered_df[
+                (filtered_df['timestamp'].dt.year == selected_date.year) &
+                (filtered_df['timestamp'].dt.month == selected_date.month)
+            ]
+            logging.info(f"After month filter ({month_filter}): {filtered_df.shape[0]} rows")
+
         # Check if filtered_df is empty
         if filtered_df.empty:
             logging.warning("Filtered DataFrame is empty after applying filters.")
-            return "No data after applying filters.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None
+            return "No data after applying filters.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
         # Log the state of filtered_df
         logging.info(f"Filtered DataFrame:\n{filtered_df.head().to_string()}")
@@ -695,6 +794,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
             future_insights = executor.submit(generate_dashboard_insights, filtered_df)
             future_usage_chart = executor.submit(create_usage_chart, filtered_df)
+            future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
             future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
             future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
             future_device_cards = executor.submit(generate_device_cards, filtered_df)
@@ -707,6 +807,7 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
             amc_reminders = f"AMC Reminders\n{amc_reminders}"
             insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
             usage_chart = future_usage_chart.result()
+            downtime_chart = future_downtime_chart.result()
             daily_log_chart = future_daily_log_chart.result()
             weekly_uptime_chart = future_weekly_uptime_chart.result()
             device_cards = future_device_cards.result()
@@ -728,20 +829,23 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
         save_to_salesforce(filtered_df, reminders_df, summary, anomalies, amc_reminders, insights)
         
         # Generate PDF with updated content
-        pdf_file = generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart)
+        pdf_file = generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, filtered_df, month_filter)
 
         elapsed_time = (datetime.now() - start_time).total_seconds()
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        return summary, preview, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, anomalies, amc_reminders, insights, pdf_file
+        if elapsed_time > 10:
+            logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
+
+        return (summary, preview, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error: {str(e)}", None, None, '<p>Error processing data.</p>', None, None, None, None, None, None, None
+        return f"Error: {str(e)}", None, None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
 # Dynamically update filter options (if columns exist in the uploaded CSV)
 def update_filters(file_obj):
     if not file_obj:
         logging.info("No file uploaded for filter update, returning default options.")
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
     
     try:
         logging.info(f"Attempting to read CSV file: {file_obj.name}")
@@ -749,6 +853,7 @@ def update_filters(file_obj):
         with open(file_obj.name, 'rb') as f:
             csv_content = f.read().decode('utf-8')
         df = pd.read_csv(io.StringIO(csv_content))
+        df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
         logging.info(f"CSV file read successfully. Columns found: {list(df.columns)}")
         
         # Lab site options
@@ -769,11 +874,21 @@ def update_filters(file_obj):
         else:
             logging.warning("Column 'equipment_type' not found in CSV.")
         
-        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All')
+        # Month options based on timestamp
+        month_options = ['All']
+        if 'timestamp' in df.columns:
+            df['month_year'] = df['timestamp'].dt.strftime('%B %Y')
+            unique_months = df['month_year'].dropna().unique().tolist()
+            month_options.extend(sorted(unique_months))
+            logging.info(f"Month options extracted: {month_options}")
+        else:
+            logging.warning("Column 'timestamp' not found in CSV.")
+
+        return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
     except Exception as e:
         logging.error(f"Failed to update filters: {str(e)}")
         # Fallback: return default options
-        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
+        return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
 
 # Gradio Interface
 try:
@@ -787,7 +902,9 @@ try:
         .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
     """) as iface:
         gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
-        gr.Markdown("Upload a CSV file to analyze.")
+        gr.Markdown("Upload a CSV file to analyze. Dashboard refreshes every 5 seconds if the file changes.")
+
+        last_modified_state = gr.State(value=None)
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -816,6 +933,12 @@ try:
                         value=[-30, 0],  # Default: last 30 days
                         info="Select the range of days relative to today (e.g., -30 to 0 for the last 30 days)."
                     )
+                    month_filter = gr.Dropdown(
+                        label="Select Month for Report",
+                        choices=['All'],
+                        value='All',
+                        interactive=True
+                    )
                 
                 submit_button = gr.Button("Analyze", variant="primary")
 
@@ -839,6 +962,8 @@ try:
                         gr.Markdown("### Charts")
                         with gr.Tab("Usage Hours per Device"):
                             usage_chart_output = gr.Plot()
+                        with gr.Tab("Downtime per Device"):
+                            downtime_chart_output = gr.Plot()
                         with gr.Tab("Daily Log Trends"):
                             daily_log_trends_output = gr.Plot()
                         with gr.Tab("Weekly Uptime Percentage"):
@@ -860,23 +985,52 @@ try:
                         
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Export Report")
-                        pdf_output = gr.File(label="Download Analysis Report as PDF")
+                        pdf_output = gr.File(label="Download Monthly Status Report as PDF")
 
         # Update filters when a new file is uploaded
         file_input.change(
             fn=update_filters,
             inputs=[file_input],
-            outputs=[lab_site_filter, equipment_type_filter],
+            outputs=[lab_site_filter, equipment_type_filter, month_filter],
             queue=False  # Disable queue to ensure immediate update
         )
 
+        # Periodic update to check for file changes
+        def update_dashboard(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
+            outputs = process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state)
+            return outputs
+
+        gr.Timer(
+            fn=update_dashboard,
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, month_filter, last_modified_state],
+            outputs=[
+                summary_output,
+                preview_output,
+                usage_chart_output,
+                device_cards_output,
+                daily_log_trends_output,
+                weekly_uptime_output,
+                anomaly_alerts_output,
+                downtime_chart_output,
+                anomaly_output,
+                amc_output,
+                insights_output,
+                pdf_output,
+                last_modified_state
+            ],
+            value=5,  # Check every 5 seconds
+            _js="() => { return window.setInterval(() => { return true; }, 5000); }"
+        )
+
         submit_button.click(
             fn=process_logs,
             inputs=[
                 file_input,
                 lab_site_filter,
                 equipment_type_filter,
-                date_range_filter
+                date_range_filter,
+                month_filter,
+                last_modified_state
             ],
             outputs=[
                 summary_output,
@@ -886,10 +1040,12 @@ try:
                 daily_log_trends_output,
                 weekly_uptime_output,
                 anomaly_alerts_output,
+                downtime_chart_output,
                 anomaly_output,
                 amc_output,
                 insights_output,
-                pdf_output
+                pdf_output,
+                last_modified_state
             ]
         )