Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,67 +1,233 @@
|
|
| 1 |
-
"""
|
| 2 |
-
LabOps Log Analyzer Dashboard with CSV file upload, PDF generation, and email alerts
|
| 3 |
-
"""
|
| 4 |
import gradio as gr
|
| 5 |
import pandas as pd
|
| 6 |
from datetime import datetime, timedelta
|
| 7 |
import logging
|
| 8 |
import plotly.express as px
|
| 9 |
from sklearn.ensemble import IsolationForest
|
|
|
|
|
|
|
| 10 |
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
| 11 |
import os
|
|
|
|
| 12 |
import io
|
| 13 |
-
import
|
| 14 |
-
from email.mime.text import MIMEText
|
| 15 |
-
from email.mime.multipart import MIMEMultipart
|
| 16 |
-
from email.mime.application import MIMEApplication
|
| 17 |
|
| 18 |
# Configure logging
|
| 19 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
# Try to import reportlab
|
| 22 |
try:
|
| 23 |
from reportlab.lib.pagesizes import letter
|
| 24 |
-
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
|
| 25 |
from reportlab.lib.styles import getSampleStyleSheet
|
|
|
|
| 26 |
reportlab_available = True
|
| 27 |
logging.info("reportlab module successfully imported")
|
| 28 |
except ImportError:
|
| 29 |
logging.warning("reportlab module not found. PDF generation disabled.")
|
| 30 |
reportlab_available = False
|
| 31 |
|
| 32 |
-
#
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
try:
|
| 35 |
total_devices = df["device_id"].nunique()
|
| 36 |
most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
|
| 37 |
-
|
| 38 |
-
summary =
|
| 39 |
-
|
| 40 |
-
return summary, f"Insights: {insights}"
|
| 41 |
except Exception as e:
|
| 42 |
-
logging.error(f"Summary
|
| 43 |
-
return f"Failed to generate summary: {str(e)}"
|
| 44 |
|
| 45 |
# Anomaly detection
|
| 46 |
def detect_anomalies(df):
|
| 47 |
try:
|
| 48 |
if "usage_hours" not in df.columns or "downtime" not in df.columns:
|
| 49 |
return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
|
| 50 |
-
if len(df) > 1000:
|
| 51 |
-
df = df.sample(n=1000, random_state=42)
|
| 52 |
features = df[["usage_hours", "downtime"]].fillna(0)
|
| 53 |
-
|
|
|
|
|
|
|
| 54 |
df["anomaly"] = iso_forest.fit_predict(features)
|
| 55 |
anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
|
| 56 |
if anomalies.empty:
|
| 57 |
return "No anomalies detected.", anomalies
|
| 58 |
-
|
| 59 |
-
for _, row in anomalies.head(5).iterrows():
|
| 60 |
-
anomaly_lines.append(
|
| 61 |
-
f"- Device ID: {row['device_id']}, Usage Hours: {row['usage_hours']}, "
|
| 62 |
-
f"Downtime: {row['downtime']}, Timestamp: {row['timestamp']}"
|
| 63 |
-
)
|
| 64 |
-
return "\n".join(anomaly_lines), anomalies
|
| 65 |
except Exception as e:
|
| 66 |
logging.error(f"Anomaly detection failed: {str(e)}")
|
| 67 |
return f"Anomaly detection failed: {str(e)}", pd.DataFrame()
|
|
@@ -69,387 +235,153 @@ def detect_anomalies(df):
|
|
| 69 |
# AMC reminders
|
| 70 |
def check_amc_reminders(df, current_date):
|
| 71 |
try:
|
| 72 |
-
logging.info(f"Input DataFrame for AMC reminders:\n{df.head().to_string()}")
|
| 73 |
if "device_id" not in df.columns or "amc_date" not in df.columns:
|
| 74 |
-
logging.warning("Missing 'device_id' or 'amc_date' columns for AMC reminders.")
|
| 75 |
return "AMC reminders require 'device_id' and 'amc_date' columns.", pd.DataFrame()
|
| 76 |
-
|
| 77 |
df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
|
| 78 |
-
|
| 79 |
-
logging.info("Localizing naive AMC dates to IST")
|
| 80 |
-
df["amc_date"] = df["amc_date"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
|
| 81 |
-
|
| 82 |
-
current_date = pd.to_datetime(current_date).tz_localize('Asia/Kolkata')
|
| 83 |
-
logging.info(f"Current date for AMC check: {current_date}")
|
| 84 |
-
|
| 85 |
df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
|
| 86 |
-
logging.info(f"Days to AMC:\n{df[['device_id', 'amc_date', 'days_to_amc']].to_string()}")
|
| 87 |
-
|
| 88 |
reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
|
| 89 |
if reminders.empty:
|
| 90 |
-
logging.info("No AMC reminders found within the next 30 days.")
|
| 91 |
return "No AMC reminders due within the next 30 days.", reminders
|
| 92 |
-
|
| 93 |
-
reminder_lines = ["Upcoming AMC Reminders:"]
|
| 94 |
-
for _, row in reminders.head(5).iterrows():
|
| 95 |
-
reminder_lines.append(f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}")
|
| 96 |
-
logging.info(f"Found {len(reminders)} AMC reminders: {reminder_lines}")
|
| 97 |
-
return "\n".join(reminder_lines), reminders
|
| 98 |
except Exception as e:
|
| 99 |
logging.error(f"AMC reminder generation failed: {str(e)}")
|
| 100 |
return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
# Create usage chart
|
| 103 |
-
def create_usage_chart(
|
| 104 |
try:
|
| 105 |
-
|
| 106 |
-
logging.info(f"Usage data for chart: {usage_data.to_string()}")
|
| 107 |
-
if usage_data.empty:
|
| 108 |
-
logging.warning("Usage data is empty.")
|
| 109 |
return None
|
|
|
|
| 110 |
if len(usage_data) > 5:
|
| 111 |
usage_data = usage_data.nlargest(5, "usage_hours")
|
| 112 |
-
|
| 113 |
-
q75, q25 = usage_data["usage_hours"].quantile([0.75, 0.25])
|
| 114 |
-
iqr = q75 - q25
|
| 115 |
-
spike_threshold = q75 + 1.5 * iqr
|
| 116 |
-
usage_data["color"] = usage_data["usage_hours"].apply(
|
| 117 |
-
lambda x: "red" if x > spike_threshold else "teal"
|
| 118 |
-
)
|
| 119 |
-
|
| 120 |
fig = px.bar(
|
| 121 |
usage_data,
|
| 122 |
x="device_id",
|
| 123 |
y="usage_hours",
|
| 124 |
-
title="Usage Hours per Device
|
| 125 |
-
labels={"device_id": "Device ID", "usage_hours": "Usage Hours"}
|
| 126 |
-
color="color",
|
| 127 |
-
color_discrete_map={"teal": "#4ECDC4", "red": "#FF0000"}
|
| 128 |
-
)
|
| 129 |
-
fig.update_traces(
|
| 130 |
-
marker_line_color='#333333',
|
| 131 |
-
marker_line_width=1.5,
|
| 132 |
-
opacity=0.9
|
| 133 |
-
)
|
| 134 |
-
fig.update_layout(
|
| 135 |
-
title_font=dict(size=18, family="Arial", color="#333333"),
|
| 136 |
-
font=dict(family="Arial", size=12, color="#333333"),
|
| 137 |
-
plot_bgcolor="white",
|
| 138 |
-
paper_bgcolor="white",
|
| 139 |
-
margin=dict(l=30, r=30, t=50, b=30),
|
| 140 |
-
xaxis=dict(
|
| 141 |
-
title="Device ID",
|
| 142 |
-
showgrid=False,
|
| 143 |
-
tickangle=45,
|
| 144 |
-
title_font=dict(size=14),
|
| 145 |
-
tickfont=dict(size=12)
|
| 146 |
-
),
|
| 147 |
-
yaxis=dict(
|
| 148 |
-
title="Usage Hours",
|
| 149 |
-
gridcolor="#E5E5E5",
|
| 150 |
-
gridwidth=1,
|
| 151 |
-
title_font=dict(size=14),
|
| 152 |
-
tickfont=dict(size=12)
|
| 153 |
-
),
|
| 154 |
-
showlegend=False,
|
| 155 |
-
bargap=0.2
|
| 156 |
)
|
|
|
|
| 157 |
return fig
|
| 158 |
except Exception as e:
|
| 159 |
logging.error(f"Failed to create usage chart: {str(e)}")
|
| 160 |
return None
|
| 161 |
|
| 162 |
-
# Create downtime chart
|
| 163 |
-
def create_downtime_chart(
|
| 164 |
try:
|
| 165 |
-
downtime_data =
|
| 166 |
-
logging.info(f"Downtime data for chart: {downtime_data.to_string()}")
|
| 167 |
-
if downtime_data.empty:
|
| 168 |
-
logging.warning("Downtime data is empty.")
|
| 169 |
-
return None
|
| 170 |
if len(downtime_data) > 5:
|
| 171 |
downtime_data = downtime_data.nlargest(5, "downtime")
|
| 172 |
-
|
| 173 |
-
q75, q25 = downtime_data["downtime"].quantile([0.75, 0.25])
|
| 174 |
-
iqr = q75 - q25
|
| 175 |
-
spike_threshold = q75 + 1.5 * iqr
|
| 176 |
-
downtime_data["color"] = downtime_data["downtime"].apply(
|
| 177 |
-
lambda x: "red" if x > spike_threshold else "green"
|
| 178 |
-
)
|
| 179 |
-
|
| 180 |
fig = px.bar(
|
| 181 |
downtime_data,
|
| 182 |
x="device_id",
|
| 183 |
y="downtime",
|
| 184 |
-
title="Downtime per Device
|
| 185 |
-
labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"}
|
| 186 |
-
color="color",
|
| 187 |
-
color_discrete_map={"green": "#96CEB4", "red": "#FF0000"}
|
| 188 |
-
)
|
| 189 |
-
fig.update_traces(
|
| 190 |
-
marker_line_color='#333333',
|
| 191 |
-
marker_line_width=1.5,
|
| 192 |
-
opacity=0.9
|
| 193 |
-
)
|
| 194 |
-
fig.update_layout(
|
| 195 |
-
title_font=dict(size=18, family="Arial", color="#333333"),
|
| 196 |
-
font=dict(family="Arial", size=12, color="#333333"),
|
| 197 |
-
plot_bgcolor="white",
|
| 198 |
-
paper_bgcolor="white",
|
| 199 |
-
margin=dict(l=30, r=30, t=50, b=30),
|
| 200 |
-
xaxis=dict(
|
| 201 |
-
title="Device ID",
|
| 202 |
-
showgrid=False,
|
| 203 |
-
tickangle=45,
|
| 204 |
-
title_font=dict(size=14),
|
| 205 |
-
tickfont=dict(size=12)
|
| 206 |
-
),
|
| 207 |
-
yaxis=dict(
|
| 208 |
-
title="Downtime (Hours)",
|
| 209 |
-
gridcolor="#E5E5E5",
|
| 210 |
-
gridwidth=1,
|
| 211 |
-
title_font=dict(size=14),
|
| 212 |
-
tickfont=dict(size=12)
|
| 213 |
-
),
|
| 214 |
-
showlegend=False,
|
| 215 |
-
bargap=0.2
|
| 216 |
)
|
|
|
|
| 217 |
return fig
|
| 218 |
except Exception as e:
|
| 219 |
logging.error(f"Failed to create downtime chart: {str(e)}")
|
| 220 |
return None
|
| 221 |
|
| 222 |
-
# Create
|
| 223 |
def create_daily_log_trends_chart(df):
|
| 224 |
try:
|
| 225 |
-
if df.empty or 'timestamp' not in df.columns:
|
| 226 |
-
logging.warning("DataFrame is empty or missing 'timestamp' column for Daily Log Trends.")
|
| 227 |
-
return None
|
| 228 |
-
|
| 229 |
-
# Group by date to count logs per day
|
| 230 |
df['date'] = df['timestamp'].dt.date
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
log_counts,
|
| 235 |
x='date',
|
| 236 |
y='log_count',
|
| 237 |
title="Daily Log Trends",
|
| 238 |
labels={"date": "Date", "log_count": "Number of Logs"}
|
| 239 |
)
|
| 240 |
-
|
| 241 |
-
fig.update_traces(
|
| 242 |
-
fill='tozeroy',
|
| 243 |
-
line_color='#4ECDC4',
|
| 244 |
-
line_width=2,
|
| 245 |
-
mode='lines+markers',
|
| 246 |
-
marker=dict(size=8, color='#4ECDC4', line=dict(width=1, color='#333333')),
|
| 247 |
-
fillcolor='rgba(78, 205, 196, 0.3)' # Gradient fill with transparency
|
| 248 |
-
)
|
| 249 |
-
fig.update_layout(
|
| 250 |
-
title_font=dict(size=18, family="Arial", color="#333333"),
|
| 251 |
-
font=dict(family="Arial", size=12, color="#333333"),
|
| 252 |
-
plot_bgcolor="white",
|
| 253 |
-
paper_bgcolor="white",
|
| 254 |
-
margin=dict(l=30, r=30, t=50, b=30),
|
| 255 |
-
xaxis=dict(
|
| 256 |
-
title="Date",
|
| 257 |
-
showgrid=False,
|
| 258 |
-
title_font=dict(size=14),
|
| 259 |
-
tickfont=dict(size=12)
|
| 260 |
-
),
|
| 261 |
-
yaxis=dict(
|
| 262 |
-
title="Number of Logs",
|
| 263 |
-
gridcolor="#E5E5E5",
|
| 264 |
-
gridwidth=1,
|
| 265 |
-
title_font=dict(size=14),
|
| 266 |
-
tickfont=dict(size=12)
|
| 267 |
-
)
|
| 268 |
-
)
|
| 269 |
return fig
|
| 270 |
except Exception as e:
|
| 271 |
-
logging.error(f"Failed to create
|
| 272 |
return None
|
| 273 |
|
| 274 |
-
# Create
|
| 275 |
def create_weekly_uptime_chart(df):
|
| 276 |
try:
|
| 277 |
-
|
| 278 |
-
logging.warning("DataFrame is empty or missing required columns for Weekly Uptime Percentage.")
|
| 279 |
-
return None
|
| 280 |
-
|
| 281 |
-
logging.info(f"DataFrame for Weekly Uptime:\n{df[['timestamp', 'downtime']].to_string()}")
|
| 282 |
-
|
| 283 |
-
# Group by week (handle pandas 2.x compatibility)
|
| 284 |
-
try:
|
| 285 |
-
df['week'] = df['timestamp'].dt.isocalendar().week
|
| 286 |
-
except AttributeError:
|
| 287 |
-
# For pandas 2.x, use .dt.weekofyear or manual calculation
|
| 288 |
-
df['week'] = df['timestamp'].dt.isocalendar()['week']
|
| 289 |
df['year'] = df['timestamp'].dt.year
|
| 290 |
weekly_data = df.groupby(['year', 'week']).agg({
|
|
|
|
| 291 |
'downtime': 'sum'
|
| 292 |
}).reset_index()
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
# Calculate uptime percentage (assuming 24*7 = 168 hours per week)
|
| 297 |
-
total_hours_per_week = 168
|
| 298 |
-
weekly_data['uptime_percentage'] = ((total_hours_per_week - weekly_data['downtime']) / total_hours_per_week) * 100
|
| 299 |
-
weekly_data['uptime_percentage'] = weekly_data['uptime_percentage'].clip(0, 100) # Ensure percentage is between 0 and 100
|
| 300 |
-
weekly_data['week_label'] = weekly_data.apply(lambda x: f"{x['year']}-W{x['week']:02d}", axis=1)
|
| 301 |
-
|
| 302 |
-
if weekly_data.empty:
|
| 303 |
-
logging.warning("No weekly data available for Weekly Uptime Percentage chart.")
|
| 304 |
-
return None
|
| 305 |
-
|
| 306 |
fig = px.bar(
|
| 307 |
weekly_data,
|
| 308 |
-
x='
|
| 309 |
-
y='
|
| 310 |
title="Weekly Uptime Percentage",
|
| 311 |
-
labels={"
|
| 312 |
-
color='uptime_percentage',
|
| 313 |
-
color_continuous_scale=['#FF0000', '#96CEB4']
|
| 314 |
-
)
|
| 315 |
-
fig.update_traces(
|
| 316 |
-
marker_line_color='#333333',
|
| 317 |
-
marker_line_width=1.5,
|
| 318 |
-
opacity=0.9
|
| 319 |
-
)
|
| 320 |
-
fig.update_layout(
|
| 321 |
-
title_font=dict(size=18, family="Arial", color="#333333"),
|
| 322 |
-
font=dict(family="Arial", size=12, color="#333333"),
|
| 323 |
-
plot_bgcolor="white",
|
| 324 |
-
paper_bgcolor="white",
|
| 325 |
-
margin=dict(l=30, r=30, t=50, b=30),
|
| 326 |
-
xaxis=dict(
|
| 327 |
-
title="Week",
|
| 328 |
-
showgrid=False,
|
| 329 |
-
tickangle=45,
|
| 330 |
-
title_font=dict(size=14),
|
| 331 |
-
tickfont=dict(size=12)
|
| 332 |
-
),
|
| 333 |
-
yaxis=dict(
|
| 334 |
-
title="Uptime Percentage (%)",
|
| 335 |
-
gridcolor="#E5E5E5",
|
| 336 |
-
gridwidth=1,
|
| 337 |
-
title_font=dict(size=14),
|
| 338 |
-
tickfont=dict(size=12)
|
| 339 |
-
),
|
| 340 |
-
showlegend=False,
|
| 341 |
-
bargap=0.2
|
| 342 |
)
|
|
|
|
| 343 |
return fig
|
| 344 |
except Exception as e:
|
| 345 |
-
logging.error(f"Failed to create
|
| 346 |
return None
|
| 347 |
|
| 348 |
-
# Create
|
| 349 |
-
def create_anomaly_alerts_chart(
|
| 350 |
try:
|
| 351 |
-
if
|
| 352 |
-
logging.warning("DataFrame or anomalies DataFrame is empty for Anomaly Alerts chart.")
|
| 353 |
return None
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
df['is_anomaly'] = df.index.isin(anomalies_df.index)
|
| 357 |
-
df['color'] = df['is_anomaly'].map({True: 'red', False: 'blue'})
|
| 358 |
-
|
| 359 |
fig = px.scatter(
|
| 360 |
-
|
| 361 |
-
x='
|
| 362 |
-
y='
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
title="Anomaly Alerts (Red = Anomaly)",
|
| 366 |
-
labels={"usage_hours": "Usage Hours", "downtime": "Downtime (Hours)"},
|
| 367 |
-
color_discrete_map={'blue': '#4ECDC4', 'red': '#FF0000'}
|
| 368 |
-
)
|
| 369 |
-
fig.update_traces(
|
| 370 |
-
marker=dict(
|
| 371 |
-
sizemode='area',
|
| 372 |
-
sizeref=0.1, # Adjust bubble size scaling
|
| 373 |
-
line=dict(width=1, color='#333333')
|
| 374 |
-
),
|
| 375 |
-
opacity=0.7
|
| 376 |
-
)
|
| 377 |
-
fig.update_layout(
|
| 378 |
-
title_font=dict(size=18, family="Arial", color="#333333"),
|
| 379 |
-
font=dict(family="Arial", size=12, color="#333333"),
|
| 380 |
-
plot_bgcolor="white",
|
| 381 |
-
paper_bgcolor="white",
|
| 382 |
-
margin=dict(l=30, r=30, t=50, b=30),
|
| 383 |
-
xaxis=dict(
|
| 384 |
-
title="Usage Hours",
|
| 385 |
-
showgrid=False,
|
| 386 |
-
title_font=dict(size=14),
|
| 387 |
-
tickfont=dict(size=12)
|
| 388 |
-
),
|
| 389 |
-
yaxis=dict(
|
| 390 |
-
title="Downtime (Hours)",
|
| 391 |
-
gridcolor="#E5E5E5",
|
| 392 |
-
gridwidth=1,
|
| 393 |
-
title_font=dict(size=14),
|
| 394 |
-
tickfont=dict(size=12)
|
| 395 |
-
),
|
| 396 |
-
showlegend=False
|
| 397 |
)
|
|
|
|
| 398 |
return fig
|
| 399 |
except Exception as e:
|
| 400 |
-
logging.error(f"Failed to create
|
| 401 |
return None
|
| 402 |
|
| 403 |
-
# Generate
|
| 404 |
def generate_device_cards(df):
|
| 405 |
try:
|
| 406 |
if df.empty:
|
| 407 |
-
logging.warning("DataFrame is empty in generate_device_cards.")
|
| 408 |
return '<p>No devices available to display.</p>'
|
| 409 |
-
|
| 410 |
-
required_columns = ['device_id', 'status', 'timestamp']
|
| 411 |
-
missing_columns = [col for col in required_columns if col not in df.columns]
|
| 412 |
-
if missing_columns:
|
| 413 |
-
logging.error(f"Missing required columns in DataFrame: {missing_columns}")
|
| 414 |
-
return f'<p>Error: Missing required columns: {missing_columns}</p>'
|
| 415 |
-
|
| 416 |
-
if df['timestamp'].isna().all():
|
| 417 |
-
logging.warning("All timestamps are NaT. Cannot generate device cards.")
|
| 418 |
-
return '<p>Error: All timestamps are invalid.</p>'
|
| 419 |
-
|
| 420 |
-
df_clean = df.dropna(subset=['timestamp']).copy()
|
| 421 |
-
if df_clean.empty:
|
| 422 |
-
logging.warning("DataFrame is empty after dropping NaT timestamps.")
|
| 423 |
-
return '<p>No valid timestamps available to display.</p>'
|
| 424 |
-
|
| 425 |
-
device_stats = df_clean.groupby('device_id').agg({
|
| 426 |
'status': 'last',
|
| 427 |
'timestamp': 'max',
|
| 428 |
}).reset_index()
|
| 429 |
-
|
| 430 |
-
counts = df_clean.groupby('device_id').size().reset_index(name='count')
|
| 431 |
-
device_stats = device_stats.merge(counts, on='device_id')
|
| 432 |
-
|
| 433 |
-
# Limit to top 10 devices by count
|
| 434 |
-
device_stats = device_stats.nlargest(10, 'count')
|
| 435 |
-
logging.info(f"Limited device cards to top {len(device_stats)} devices by usage count.")
|
| 436 |
-
|
| 437 |
device_stats['health'] = device_stats['status'].map({
|
| 438 |
'Active': 'Healthy',
|
| 439 |
'Inactive': 'Unhealthy',
|
| 440 |
'Pending': 'Warning'
|
| 441 |
}).fillna('Unknown')
|
| 442 |
-
|
| 443 |
cards_html = '<div style="display: flex; flex-wrap: wrap; gap: 20px;">'
|
| 444 |
for _, row in device_stats.iterrows():
|
| 445 |
-
health_color = {
|
| 446 |
-
'Healthy': 'green',
|
| 447 |
-
'Unhealthy': 'red',
|
| 448 |
-
'Warning': 'orange',
|
| 449 |
-
'Unknown': 'gray'
|
| 450 |
-
}.get(row['health'], 'gray')
|
| 451 |
timestamp_str = str(row['timestamp']) if pd.notna(row['timestamp']) else 'Unknown'
|
| 452 |
-
|
| 453 |
<div style="border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; width: 200px;">
|
| 454 |
<h4>Device: {row['device_id']}</h4>
|
| 455 |
<p><b>Health:</b> <span style="color: {health_color}">{row['health']}</span></p>
|
|
@@ -457,57 +389,50 @@ def generate_device_cards(df):
|
|
| 457 |
<p><b>Last Log:</b> {timestamp_str}</p>
|
| 458 |
</div>
|
| 459 |
"""
|
| 460 |
-
cards_html += card
|
| 461 |
cards_html += '</div>'
|
| 462 |
-
logging.info("Device cards generated successfully")
|
| 463 |
return cards_html
|
| 464 |
except Exception as e:
|
| 465 |
-
logging.error(f"Failed to generate device cards: {str(e)}"
|
| 466 |
return f'<p>Error generating device cards: {str(e)}</p>'
|
| 467 |
|
| 468 |
-
# Generate monthly status
|
| 469 |
def generate_monthly_status(df, selected_month):
|
| 470 |
try:
|
| 471 |
total_devices = df['device_id'].nunique()
|
| 472 |
total_usage_hours = df['usage_hours'].sum()
|
| 473 |
total_downtime = df['downtime'].sum()
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
summary = f"""
|
| 478 |
Monthly Status for {selected_month}:
|
| 479 |
- Total Devices: {total_devices}
|
| 480 |
- Total Usage Hours: {total_usage_hours:.2f}
|
| 481 |
- Total Downtime Hours: {total_downtime:.2f}
|
| 482 |
-
- Average Usage per Device: {
|
| 483 |
-
- Average Downtime per Device: {
|
| 484 |
"""
|
| 485 |
-
return summary
|
| 486 |
except Exception as e:
|
| 487 |
logging.error(f"Failed to generate monthly status: {str(e)}")
|
| 488 |
return f"Failed to generate monthly status: {str(e)}"
|
| 489 |
|
| 490 |
# Generate PDF content
|
| 491 |
-
def generate_pdf_content(summary,
|
| 492 |
if not reportlab_available:
|
| 493 |
-
logging.error("reportlab not available. PDF generation is disabled.")
|
| 494 |
return None
|
| 495 |
try:
|
| 496 |
-
logging.info("Starting PDF generation...")
|
| 497 |
pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
|
| 498 |
doc = SimpleDocTemplate(pdf_path, pagesize=letter)
|
| 499 |
styles = getSampleStyleSheet()
|
| 500 |
story = []
|
| 501 |
|
| 502 |
def safe_paragraph(text, style):
|
| 503 |
-
|
| 504 |
-
return Paragraph(text_str.replace('\n', '<br/>'), style) if text_str else Paragraph("", style)
|
| 505 |
|
| 506 |
story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
|
| 507 |
story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
|
| 508 |
story.append(Spacer(1, 12))
|
| 509 |
|
| 510 |
-
if selected_month != "All"
|
| 511 |
monthly_status = generate_monthly_status(df, selected_month)
|
| 512 |
story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
|
| 513 |
story.append(safe_paragraph(monthly_status, styles['Normal']))
|
|
@@ -518,11 +443,29 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
|
|
| 518 |
story.append(Spacer(1, 12))
|
| 519 |
|
| 520 |
story.append(Paragraph("Log Preview", styles['Heading2']))
|
| 521 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 522 |
story.append(Spacer(1, 12))
|
| 523 |
|
| 524 |
story.append(Paragraph("Device Cards", styles['Heading2']))
|
| 525 |
-
device_cards_text = device_cards_html.replace('<div>', '').replace('</div>', '\n').replace('<h4>', '').replace('</h4>', '\n').replace('<p>', '').replace('</p>', '\n').replace('<b>', '').replace('</b>', '').replace('<span style="color: green">', '').replace('<span style="color: red">', '').replace('<span style="color: orange">', '').replace('<span style="color: gray">', '').replace('</span>', '')
|
| 526 |
story.append(safe_paragraph(device_cards_text, styles['Normal']))
|
| 527 |
story.append(Spacer(1, 12))
|
| 528 |
|
|
@@ -538,105 +481,31 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
|
|
| 538 |
story.append(safe_paragraph(insights, styles['Normal']))
|
| 539 |
story.append(Spacer(1, 12))
|
| 540 |
|
| 541 |
-
story.append(Paragraph("
|
| 542 |
-
story.append(Paragraph("[Chart
|
| 543 |
-
story.append(Spacer(1, 12))
|
| 544 |
-
|
| 545 |
-
story.append(Paragraph("Weekly Uptime Percentage Chart", styles['Heading2']))
|
| 546 |
-
story.append(Paragraph("[Chart placeholder - see dashboard for Weekly Uptime Percentage]" if weekly_uptime_chart is None else "[Chart included in dashboard]", styles['Normal']))
|
| 547 |
-
story.append(Spacer(1, 12))
|
| 548 |
-
|
| 549 |
-
story.append(Paragraph("Anomaly Alerts Chart", styles['Heading2']))
|
| 550 |
-
story.append(Paragraph("[Chart placeholder - see dashboard for Anomaly Alerts]" if anomaly_alerts_chart is None else "[Chart included in dashboard]", styles['Normal']))
|
| 551 |
-
story.append(Spacer(1, 12))
|
| 552 |
-
|
| 553 |
-
story.append(Paragraph("Downtime Chart", styles['Heading2']))
|
| 554 |
-
story.append(Paragraph("[Chart placeholder - see dashboard for Downtime per Device]" if downtime_chart is None else "[Chart included in dashboard]", styles['Normal']))
|
| 555 |
|
| 556 |
doc.build(story)
|
| 557 |
logging.info(f"PDF generated at {pdf_path}")
|
| 558 |
return pdf_path
|
| 559 |
except Exception as e:
|
| 560 |
-
logging.error(f"Failed to generate PDF: {str(e)}"
|
| 561 |
return None
|
| 562 |
|
| 563 |
-
#
|
| 564 |
-
def send_email_alert(summary, anomalies, amc_reminders, pdf_path, recipient_email="recipient@example.com"):
|
| 565 |
-
try:
|
| 566 |
-
# Email configuration
|
| 567 |
-
sender_email = "your_email@gmail.com" # Replace with your email
|
| 568 |
-
sender_password = "your_app_password" # Replace with your app-specific password
|
| 569 |
-
smtp_server = "smtp.gmail.com"
|
| 570 |
-
smtp_port = 587
|
| 571 |
-
|
| 572 |
-
# Create email message
|
| 573 |
-
subject = "LabOps Log Analyzer Report - Analysis Completed"
|
| 574 |
-
body = f"""
|
| 575 |
-
Dear Recipient,
|
| 576 |
-
|
| 577 |
-
The LabOps Log Analyzer has completed its analysis. Below are the key findings:
|
| 578 |
-
|
| 579 |
-
**Summary:**
|
| 580 |
-
{summary}
|
| 581 |
-
|
| 582 |
-
**Anomalies Detected:**
|
| 583 |
-
{anomalies}
|
| 584 |
-
|
| 585 |
-
**AMC Reminders:**
|
| 586 |
-
{amc_reminders}
|
| 587 |
-
|
| 588 |
-
The full report is attached as a PDF for your review.
|
| 589 |
-
|
| 590 |
-
Regards,
|
| 591 |
-
LabOps Team
|
| 592 |
-
"""
|
| 593 |
-
|
| 594 |
-
msg = MIMEMultipart()
|
| 595 |
-
msg['From'] = sender_email
|
| 596 |
-
msg['To'] = recipient_email
|
| 597 |
-
msg['Subject'] = subject
|
| 598 |
-
msg.attach(MIMEText(body, 'plain'))
|
| 599 |
-
|
| 600 |
-
# Attach the PDF if it exists
|
| 601 |
-
if pdf_path and os.path.exists(pdf_path):
|
| 602 |
-
with open(pdf_path, 'rb') as f:
|
| 603 |
-
pdf_attachment = MIMEApplication(f.read(), _subtype="pdf")
|
| 604 |
-
pdf_attachment.add_header(
|
| 605 |
-
'Content-Disposition', 'attachment', filename=os.path.basename(pdf_path)
|
| 606 |
-
)
|
| 607 |
-
msg.attach(pdf_attachment)
|
| 608 |
-
logging.info(f"Attached PDF to email: {pdf_path}")
|
| 609 |
-
else:
|
| 610 |
-
logging.warning("No PDF file to attach to email.")
|
| 611 |
-
|
| 612 |
-
# Send the email
|
| 613 |
-
with smtplib.SMTP(smtp_server, smtp_port) as server:
|
| 614 |
-
server.starttls()
|
| 615 |
-
server.login(sender_email, sender_password)
|
| 616 |
-
server.sendmail(sender_email, recipient_email, msg.as_string())
|
| 617 |
-
|
| 618 |
-
logging.info(f"Email alert sent to {recipient_email}")
|
| 619 |
-
except Exception as e:
|
| 620 |
-
logging.error(f"Failed to send email alert: {str(e)}")
|
| 621 |
-
|
| 622 |
-
# Main Gradio function
|
| 623 |
async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
|
|
|
|
| 624 |
try:
|
| 625 |
-
start_time = datetime.now()
|
| 626 |
-
|
| 627 |
if not file_obj:
|
| 628 |
-
return "No file uploaded.",
|
| 629 |
-
|
| 630 |
file_path = file_obj.name
|
| 631 |
current_modified_time = os.path.getmtime(file_path)
|
| 632 |
-
|
| 633 |
if last_modified_state and current_modified_time == last_modified_state:
|
| 634 |
-
return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state
|
| 635 |
|
| 636 |
-
logging.info(f"Processing file: {file_path}
|
| 637 |
-
|
| 638 |
if not file_path.endswith(".csv"):
|
| 639 |
-
return "Please upload a CSV file.",
|
| 640 |
|
| 641 |
required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
|
| 642 |
dtypes = {
|
|
@@ -648,235 +517,96 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
|
|
| 648 |
"amc_date": "string"
|
| 649 |
}
|
| 650 |
df = pd.read_csv(file_path, dtype=dtypes)
|
| 651 |
-
# Downsample early if dataset is too large
|
| 652 |
-
if len(df) > 5000:
|
| 653 |
-
df = df.sample(n=5000, random_state=42)
|
| 654 |
-
logging.info(f"Downsampled DataFrame to 5,000 rows immediately after loading.")
|
| 655 |
missing_columns = [col for col in required_columns if col not in df.columns]
|
| 656 |
if missing_columns:
|
| 657 |
-
return f"Missing columns: {missing_columns}",
|
| 658 |
-
|
| 659 |
df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
|
| 660 |
df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
|
| 661 |
if df["timestamp"].dt.tz is None:
|
| 662 |
-
logging.info("Localizing naive timestamps to IST")
|
| 663 |
df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
|
| 664 |
if df.empty:
|
| 665 |
-
return "No data available.",
|
| 666 |
-
|
| 667 |
-
logging.info(f"DataFrame before filtering:\n{df.head().to_string()}")
|
| 668 |
-
|
| 669 |
-
# Apply filters directly on df
|
| 670 |
-
filtered_df = df
|
| 671 |
|
|
|
|
|
|
|
| 672 |
if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
|
| 673 |
filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
|
| 674 |
-
logging.info(f"After lab_site filter ({lab_site_filter}): {filtered_df.shape[0]} rows")
|
| 675 |
-
|
| 676 |
if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
|
| 677 |
filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
|
| 678 |
-
logging.info(f"After equipment_type filter ({equipment_type_filter}): {filtered_df.shape[0]} rows")
|
| 679 |
-
|
| 680 |
if date_range and len(date_range) == 2:
|
| 681 |
days_start, days_end = date_range
|
| 682 |
today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
|
| 683 |
start_date = today + pd.Timedelta(days=days_start)
|
| 684 |
-
end_date = today + pd.Timedelta(days=days_end)
|
| 685 |
-
end_date = end_date + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
|
| 686 |
-
logging.info(f"Applying date range filter: {start_date} to {end_date}")
|
| 687 |
filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
|
| 688 |
-
logging.info(f"After date range filter: {filtered_df.shape[0]} rows")
|
| 689 |
-
|
| 690 |
if month_filter and month_filter != "All":
|
| 691 |
selected_date = pd.to_datetime(month_filter, format="%B %Y")
|
| 692 |
filtered_df = filtered_df[
|
| 693 |
(filtered_df['timestamp'].dt.year == selected_date.year) &
|
| 694 |
(filtered_df['timestamp'].dt.month == selected_date.month)
|
| 695 |
]
|
| 696 |
-
logging.info(f"After month filter ({month_filter}): {filtered_df.shape[0]} rows")
|
| 697 |
|
| 698 |
if filtered_df.empty:
|
| 699 |
-
|
| 700 |
-
return "No data after applying filters.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, None, None, None, None, None, None, "No data available after applying filters."
|
| 701 |
-
|
| 702 |
-
logging.info(f"Filtered DataFrame before AMC check:\n{filtered_df[['device_id', 'amc_date']].to_string()}")
|
| 703 |
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
# Pre-aggregate data for charts
|
| 709 |
-
agg_data = {
|
| 710 |
-
'usage_per_device': filtered_df.groupby("device_id")["usage_hours"].sum().reset_index(),
|
| 711 |
-
'downtime_per_device': filtered_df.groupby("device_id")["downtime"].sum().reset_index(),
|
| 712 |
-
}
|
| 713 |
|
| 714 |
# Run tasks concurrently
|
| 715 |
-
with ThreadPoolExecutor(max_workers=
|
| 716 |
-
|
| 717 |
future_anomalies = executor.submit(detect_anomalies, filtered_df)
|
| 718 |
future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
|
| 719 |
-
|
| 720 |
-
|
|
|
|
| 721 |
future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
|
| 722 |
future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
|
|
|
|
| 723 |
future_device_cards = executor.submit(generate_device_cards, filtered_df)
|
|
|
|
| 724 |
|
| 725 |
-
summary
|
| 726 |
-
summary = f"Step 1: Summary Report\n{summary}"
|
| 727 |
-
insights = f"Dashboard Insights (AI)\n{insights}"
|
| 728 |
anomalies, anomalies_df = future_anomalies.result()
|
| 729 |
anomalies = f"Anomaly Detection\n{anomalies}"
|
| 730 |
amc_reminders, reminders_df = future_amc.result()
|
| 731 |
amc_reminders = f"AMC Reminders\n{amc_reminders}"
|
|
|
|
| 732 |
usage_chart = future_usage_chart.result()
|
| 733 |
downtime_chart = future_downtime_chart.result()
|
| 734 |
daily_log_chart = future_daily_log_chart.result()
|
| 735 |
weekly_uptime_chart = future_weekly_uptime_chart.result()
|
|
|
|
| 736 |
device_cards = future_device_cards.result()
|
| 737 |
|
| 738 |
-
|
| 739 |
-
|
| 740 |
|
| 741 |
-
|
| 742 |
-
preview_html = """
|
| 743 |
-
<style>
|
| 744 |
-
.log-preview-table {
|
| 745 |
-
width: 100%;
|
| 746 |
-
border-collapse: collapse;
|
| 747 |
-
font-family: Arial, sans-serif;
|
| 748 |
-
margin-top: 10px;
|
| 749 |
-
}
|
| 750 |
-
.log-preview-table th, .log-preview-table td {
|
| 751 |
-
border: 1px solid #ddd;
|
| 752 |
-
padding: 8px;
|
| 753 |
-
text-align: left;
|
| 754 |
-
}
|
| 755 |
-
.log-preview-table th {
|
| 756 |
-
background-color: #4ECDC4;
|
| 757 |
-
color: white;
|
| 758 |
-
}
|
| 759 |
-
.log-preview-table tr:nth-child(even) {
|
| 760 |
-
background-color: #f2f2f2;
|
| 761 |
-
}
|
| 762 |
-
.log-preview-table tr:hover {
|
| 763 |
-
background-color: #ddd;
|
| 764 |
-
}
|
| 765 |
-
</style>
|
| 766 |
-
<h3>Step 2: Log Preview (First 5 Rows)</h3>
|
| 767 |
-
<table class='log-preview-table'>
|
| 768 |
-
<thead>
|
| 769 |
-
<tr>
|
| 770 |
-
<th>Row</th>
|
| 771 |
-
<th>Device ID</th>
|
| 772 |
-
<th>Log Type</th>
|
| 773 |
-
<th>Status</th>
|
| 774 |
-
<th>Timestamp</th>
|
| 775 |
-
<th>Usage Hours</th>
|
| 776 |
-
<th>Downtime</th>
|
| 777 |
-
<th>AMC Date</th>
|
| 778 |
-
</tr>
|
| 779 |
-
</thead>
|
| 780 |
-
<tbody>
|
| 781 |
-
"""
|
| 782 |
-
if filtered_df.empty:
|
| 783 |
-
preview_html += "<tr><td colspan='8'>No data to preview.</td></tr>"
|
| 784 |
-
else:
|
| 785 |
-
for idx, row in filtered_df.head(5).iterrows():
|
| 786 |
-
preview_html += f"""
|
| 787 |
-
<tr>
|
| 788 |
-
<td>{idx + 1}</td>
|
| 789 |
-
<td>{row['device_id']}</td>
|
| 790 |
-
<td>{row['log_type']}</td>
|
| 791 |
-
<td>{row['status']}</td>
|
| 792 |
-
<td>{row['timestamp']}</td>
|
| 793 |
-
<td>{row['usage_hours']}</td>
|
| 794 |
-
<td>{row['downtime']}</td>
|
| 795 |
-
<td>{row['amc_date']}</td>
|
| 796 |
-
</tr>
|
| 797 |
-
"""
|
| 798 |
-
preview_html += """
|
| 799 |
-
</tbody>
|
| 800 |
-
</table>
|
| 801 |
-
"""
|
| 802 |
-
|
| 803 |
-
preview_lines = ["Step 2: Log Preview (First 5 Rows)"]
|
| 804 |
-
for idx, row in filtered_df.head(5).iterrows():
|
| 805 |
-
preview_lines.append(
|
| 806 |
-
f"Row {idx + 1}: Device ID: {row['device_id']}, "
|
| 807 |
-
f"Log Type: {row['log_type']}, Status: {row['status']}, "
|
| 808 |
-
f"Timestamp: {row['timestamp']}, Usage Hours: {row['usage_hours']}, "
|
| 809 |
-
f"Downtime: {row['downtime']}, AMC Date: {row['amc_date']}"
|
| 810 |
-
)
|
| 811 |
-
preview_text = "\n".join(preview_lines)
|
| 812 |
-
|
| 813 |
-
# Auto-generate PDF after analysis
|
| 814 |
-
pdf_file = None
|
| 815 |
-
status_msg = "Analysis completed successfully."
|
| 816 |
-
if all([summary, preview_text, anomalies, amc_reminders, insights, device_cards, filtered_df is not None]):
|
| 817 |
-
pdf_file = generate_pdf_content(
|
| 818 |
-
summary, preview_text, anomalies, amc_reminders, insights, device_cards,
|
| 819 |
-
daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart,
|
| 820 |
-
filtered_df, month_filter
|
| 821 |
-
)
|
| 822 |
-
if pdf_file:
|
| 823 |
-
status_msg = "Analysis completed successfully. PDF report generated and available for download."
|
| 824 |
-
else:
|
| 825 |
-
status_msg = "Analysis completed successfully, but failed to generate PDF. Check logs for details."
|
| 826 |
-
else:
|
| 827 |
-
status_msg = "Analysis completed, but some data is missing for PDF generation."
|
| 828 |
-
|
| 829 |
-
# Send email alert
|
| 830 |
-
send_email_alert(summary, anomalies, amc_reminders, pdf_file)
|
| 831 |
-
|
| 832 |
-
elapsed_time = (datetime.now() - start_time).total_seconds()
|
| 833 |
logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
|
| 834 |
if elapsed_time > 10:
|
| 835 |
logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
|
| 836 |
|
| 837 |
-
return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time
|
| 838 |
except Exception as e:
|
| 839 |
logging.error(f"Failed to process file: {str(e)}")
|
| 840 |
-
return f"Error: {str(e)}",
|
| 841 |
|
| 842 |
-
# Update
|
| 843 |
def update_filters(file_obj):
|
| 844 |
if not file_obj:
|
| 845 |
-
logging.info("No file uploaded for filter update, returning default options.")
|
| 846 |
return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
|
| 847 |
-
|
| 848 |
try:
|
| 849 |
-
logging.info(f"Attempting to read CSV file: {file_obj.name}")
|
| 850 |
with open(file_obj.name, 'rb') as f:
|
| 851 |
csv_content = f.read().decode('utf-8')
|
| 852 |
df = pd.read_csv(io.StringIO(csv_content))
|
| 853 |
df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
|
| 854 |
-
|
| 855 |
-
|
| 856 |
-
|
| 857 |
-
if '
|
| 858 |
-
unique_lab_sites = df['lab_site'].dropna().astype(str).unique().tolist()
|
| 859 |
-
lab_site_options.extend([site for site in unique_lab_sites if site.strip()])
|
| 860 |
-
logging.info(f"Lab site options extracted: {lab_site_options}")
|
| 861 |
-
else:
|
| 862 |
-
logging.warning("Column 'lab_site' not found in CSV.")
|
| 863 |
-
|
| 864 |
-
equipment_type_options = ['All']
|
| 865 |
-
if 'equipment_type' in df.columns:
|
| 866 |
-
unique_equipment_types = df['equipment_type'].dropna().astype(str).unique().tolist()
|
| 867 |
-
equipment_type_options.extend([equip for equip in unique_equipment_types if equip.strip()])
|
| 868 |
-
logging.info(f"Equipment type options extracted: {equipment_type_options}")
|
| 869 |
-
else:
|
| 870 |
-
logging.warning("Column 'equipment_type' not found in CSV.")
|
| 871 |
-
|
| 872 |
-
month_options = ['All']
|
| 873 |
-
if 'timestamp' in df.columns:
|
| 874 |
-
df['month_year'] = df['timestamp'].dt.strftime('%B %Y')
|
| 875 |
-
unique_months = df['month_year'].dropna().unique().tolist()
|
| 876 |
-
month_options.extend(sorted(unique_months))
|
| 877 |
-
logging.info(f"Month options extracted: {month_options}")
|
| 878 |
-
else:
|
| 879 |
-
logging.warning("Column 'timestamp' not found in CSV.")
|
| 880 |
|
| 881 |
return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
|
| 882 |
except Exception as e:
|
|
@@ -893,74 +623,39 @@ try:
|
|
| 893 |
.dashboard-section h3 {font-size: 18px; margin-bottom: 2px;}
|
| 894 |
.dashboard-section p {margin: 1px 0; line-height: 1.2;}
|
| 895 |
.dashboard-section ul {margin: 2px 0; padding-left: 20px;}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 896 |
""") as iface:
|
| 897 |
-
gr.Markdown("<h1>LabOps Log Analyzer Dashboard</h1>")
|
| 898 |
-
gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data.
|
| 899 |
|
| 900 |
last_modified_state = gr.State(value=None)
|
| 901 |
-
summary_state = gr.State()
|
| 902 |
-
preview_state = gr.State()
|
| 903 |
-
anomalies_state = gr.State()
|
| 904 |
-
amc_reminders_state = gr.State()
|
| 905 |
-
insights_state = gr.State()
|
| 906 |
-
device_cards_state = gr.State()
|
| 907 |
-
df_state = gr.State()
|
| 908 |
|
| 909 |
with gr.Row():
|
| 910 |
with gr.Column(scale=1):
|
| 911 |
file_input = gr.File(label="Upload Logs (CSV)", file_types=[".csv"])
|
| 912 |
-
|
| 913 |
with gr.Group():
|
| 914 |
gr.Markdown("### Filters")
|
| 915 |
-
lab_site_filter = gr.Dropdown(
|
| 916 |
-
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
interactive=True
|
| 920 |
-
)
|
| 921 |
-
equipment_type_filter = gr.Dropdown(
|
| 922 |
-
label="Equipment Type",
|
| 923 |
-
choices=['All'],
|
| 924 |
-
value='All',
|
| 925 |
-
interactive=True
|
| 926 |
-
)
|
| 927 |
-
date_range_filter = gr.Slider(
|
| 928 |
-
label="Date Range (Days from Today)",
|
| 929 |
-
minimum=-365,
|
| 930 |
-
maximum=0,
|
| 931 |
-
step=1,
|
| 932 |
-
value=[-30, 0],
|
| 933 |
-
info="Select the range of days relative to today (e.g., -30 to 0 for the last 30 days)."
|
| 934 |
-
)
|
| 935 |
-
month_filter = gr.Dropdown(
|
| 936 |
-
label="Select Month for Report",
|
| 937 |
-
choices=['All'],
|
| 938 |
-
value='All',
|
| 939 |
-
interactive=True
|
| 940 |
-
)
|
| 941 |
-
|
| 942 |
submit_button = gr.Button("Analyze", variant="primary")
|
| 943 |
|
| 944 |
with gr.Column(scale=2):
|
| 945 |
with gr.Group(elem_classes="dashboard-container"):
|
| 946 |
gr.Markdown("<div class='dashboard-title'>Analysis Results</div>")
|
| 947 |
-
|
| 948 |
-
with gr.Group(elem_classes="dashboard-section"):
|
| 949 |
-
gr.Markdown("### Status Message")
|
| 950 |
-
status_message = gr.Markdown("Please upload a CSV file and click 'Analyze' to begin.")
|
| 951 |
-
|
| 952 |
with gr.Group(elem_classes="dashboard-section"):
|
| 953 |
gr.Markdown("### Step 1: Summary Report")
|
| 954 |
summary_output = gr.Markdown()
|
| 955 |
-
|
| 956 |
with gr.Group(elem_classes="dashboard-section"):
|
| 957 |
gr.Markdown("### Step 2: Log Preview")
|
| 958 |
preview_output = gr.HTML()
|
| 959 |
-
|
| 960 |
with gr.Group(elem_classes="dashboard-section"):
|
| 961 |
gr.Markdown("### Device Cards")
|
| 962 |
device_cards_output = gr.HTML()
|
| 963 |
-
|
| 964 |
with gr.Group(elem_classes="dashboard-section"):
|
| 965 |
gr.Markdown("### Charts")
|
| 966 |
with gr.Tab("Usage Hours per Device"):
|
|
@@ -973,19 +668,15 @@ try:
|
|
| 973 |
weekly_uptime_output = gr.Plot()
|
| 974 |
with gr.Tab("Anomaly Alerts"):
|
| 975 |
anomaly_alerts_output = gr.Plot()
|
| 976 |
-
|
| 977 |
with gr.Group(elem_classes="dashboard-section"):
|
| 978 |
gr.Markdown("### Step 4: Anomaly Detection")
|
| 979 |
anomaly_output = gr.Markdown()
|
| 980 |
-
|
| 981 |
with gr.Group(elem_classes="dashboard-section"):
|
| 982 |
gr.Markdown("### Step 5: AMC Reminders")
|
| 983 |
amc_output = gr.Markdown()
|
| 984 |
-
|
| 985 |
with gr.Group(elem_classes="dashboard-section"):
|
| 986 |
gr.Markdown("### Step 6: Insights (AI)")
|
| 987 |
insights_output = gr.Markdown()
|
| 988 |
-
|
| 989 |
with gr.Group(elem_classes="dashboard-section"):
|
| 990 |
gr.Markdown("### Export Report")
|
| 991 |
pdf_output = gr.File(label="Download Monthly Status Report as PDF")
|
|
@@ -999,37 +690,8 @@ try:
|
|
| 999 |
|
| 1000 |
submit_button.click(
|
| 1001 |
fn=process_logs,
|
| 1002 |
-
inputs=[
|
| 1003 |
-
|
| 1004 |
-
lab_site_filter,
|
| 1005 |
-
equipment_type_filter,
|
| 1006 |
-
date_range_filter,
|
| 1007 |
-
month_filter,
|
| 1008 |
-
last_modified_state
|
| 1009 |
-
],
|
| 1010 |
-
outputs=[
|
| 1011 |
-
summary_output,
|
| 1012 |
-
preview_output,
|
| 1013 |
-
usage_chart_output,
|
| 1014 |
-
device_cards_output,
|
| 1015 |
-
daily_log_trends_output,
|
| 1016 |
-
weekly_uptime_output,
|
| 1017 |
-
anomaly_alerts_output,
|
| 1018 |
-
downtime_chart_output,
|
| 1019 |
-
anomaly_output,
|
| 1020 |
-
amc_output,
|
| 1021 |
-
insights_output,
|
| 1022 |
-
pdf_output,
|
| 1023 |
-
last_modified_state,
|
| 1024 |
-
summary_state,
|
| 1025 |
-
preview_state,
|
| 1026 |
-
anomalies_state,
|
| 1027 |
-
amc_reminders_state,
|
| 1028 |
-
insights_state,
|
| 1029 |
-
device_cards_state,
|
| 1030 |
-
df_state,
|
| 1031 |
-
status_message
|
| 1032 |
-
]
|
| 1033 |
)
|
| 1034 |
|
| 1035 |
logging.info("Gradio interface initialized successfully")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import gradio as gr
|
| 2 |
import pandas as pd
|
| 3 |
from datetime import datetime, timedelta
|
| 4 |
import logging
|
| 5 |
import plotly.express as px
|
| 6 |
from sklearn.ensemble import IsolationForest
|
| 7 |
+
from transformers import pipeline
|
| 8 |
+
import torch
|
| 9 |
from concurrent.futures import ThreadPoolExecutor
|
| 10 |
+
from simple_salesforce import Salesforce
|
| 11 |
import os
|
| 12 |
+
import json
|
| 13 |
import io
|
| 14 |
+
import time
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
# Configure logging
|
| 17 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
| 18 |
|
| 19 |
+
# Salesforce configuration
|
| 20 |
+
try:
|
| 21 |
+
sf = Salesforce(
|
| 22 |
+
username='multi-devicelabopsdashboard@sathkrutha.com',
|
| 23 |
+
password='Team@1234',
|
| 24 |
+
security_token=os.getenv('SF_SECURITY_TOKEN', ''),
|
| 25 |
+
domain='login'
|
| 26 |
+
)
|
| 27 |
+
logging.info("Salesforce connection established")
|
| 28 |
+
except Exception as e:
|
| 29 |
+
logging.error(f"Failed to connect to Salesforce: {str(e)}")
|
| 30 |
+
sf = None
|
| 31 |
+
|
| 32 |
# Try to import reportlab
|
| 33 |
try:
|
| 34 |
from reportlab.lib.pagesizes import letter
|
| 35 |
+
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle
|
| 36 |
from reportlab.lib.styles import getSampleStyleSheet
|
| 37 |
+
from reportlab.lib import colors
|
| 38 |
reportlab_available = True
|
| 39 |
logging.info("reportlab module successfully imported")
|
| 40 |
except ImportError:
|
| 41 |
logging.warning("reportlab module not found. PDF generation disabled.")
|
| 42 |
reportlab_available = False
|
| 43 |
|
| 44 |
+
# Preload Hugging Face model with optimization
|
| 45 |
+
logging.info("Preloading Hugging Face model...")
|
| 46 |
+
try:
|
| 47 |
+
device = 0 if torch.cuda.is_available() else -1
|
| 48 |
+
# Use a smaller model for faster inference
|
| 49 |
+
summarizer = pipeline(
|
| 50 |
+
"summarization",
|
| 51 |
+
model="t5-small",
|
| 52 |
+
device=device,
|
| 53 |
+
max_length=50,
|
| 54 |
+
min_length=10,
|
| 55 |
+
num_beams=2
|
| 56 |
+
)
|
| 57 |
+
logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
|
| 58 |
+
except Exception as e:
|
| 59 |
+
logging.error(f"Failed to preload model: {str(e)}")
|
| 60 |
+
raise e
|
| 61 |
+
|
| 62 |
+
# Cache picklist values at startup
|
| 63 |
+
def get_picklist_values(field_name):
|
| 64 |
+
if sf is None:
|
| 65 |
+
return []
|
| 66 |
+
try:
|
| 67 |
+
obj_desc = sf.SmartLog__c.describe()
|
| 68 |
+
for field in obj_desc['fields']:
|
| 69 |
+
if field['name'] == field_name:
|
| 70 |
+
return [value['value'] for value in field['picklistValues'] if value['active']]
|
| 71 |
+
return []
|
| 72 |
+
except Exception as e:
|
| 73 |
+
logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
|
| 74 |
+
return []
|
| 75 |
+
|
| 76 |
+
status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
|
| 77 |
+
log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
|
| 78 |
+
logging.info(f"Valid Status__c values: {status_values}")
|
| 79 |
+
logging.info(f"Valid Log_Type__c values: {log_type_values}")
|
| 80 |
+
|
| 81 |
+
# Map invalid picklist values
|
| 82 |
+
picklist_mapping = {
|
| 83 |
+
'Status__c': {
|
| 84 |
+
'normal': 'Active',
|
| 85 |
+
'error': 'Inactive',
|
| 86 |
+
'warning': 'Pending',
|
| 87 |
+
'ok': 'Active',
|
| 88 |
+
'failed': 'Inactive'
|
| 89 |
+
},
|
| 90 |
+
'Log_Type__c': {
|
| 91 |
+
'maint': 'Smart Log',
|
| 92 |
+
'error': 'Cell Analysis',
|
| 93 |
+
'ops': 'UV Verification',
|
| 94 |
+
'maintenance': 'Smart Log',
|
| 95 |
+
'cell': 'Cell Analysis',
|
| 96 |
+
'uv': 'UV Verification',
|
| 97 |
+
'weight log': 'Smart Log'
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
# Cache folder ID
|
| 102 |
+
def get_folder_id(folder_name):
|
| 103 |
+
if sf is None:
|
| 104 |
+
return None
|
| 105 |
+
try:
|
| 106 |
+
query = f"SELECT Id FROM Folder WHERE Name = '{folder_name}' AND Type = 'Report'"
|
| 107 |
+
result = sf.query(query)
|
| 108 |
+
if result['totalSize'] > 0:
|
| 109 |
+
folder_id = result['records'][0]['Id']
|
| 110 |
+
logging.info(f"Found folder ID for '{folder_name}': {folder_id}")
|
| 111 |
+
return folder_id
|
| 112 |
+
else:
|
| 113 |
+
logging.error(f"Folder '{folder_name}' not found in Salesforce.")
|
| 114 |
+
return None
|
| 115 |
+
except Exception as e:
|
| 116 |
+
logging.error(f"Failed to fetch folder ID for '{folder_name}': {str(e)}")
|
| 117 |
+
return None
|
| 118 |
+
|
| 119 |
+
LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
|
| 120 |
+
|
| 121 |
+
# Salesforce report creation
|
| 122 |
+
def create_salesforce_reports(df):
|
| 123 |
+
if sf is None or not LABOPS_REPORTS_FOLDER_ID:
|
| 124 |
+
return
|
| 125 |
+
try:
|
| 126 |
+
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
| 127 |
+
reports = [
|
| 128 |
+
{
|
| 129 |
+
"reportMetadata": {
|
| 130 |
+
"name": f"SmartLog_Usage_Report_{timestamp}",
|
| 131 |
+
"developerName": f"SmartLog_Usage_Report_{timestamp}",
|
| 132 |
+
"reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
|
| 133 |
+
"reportFormat": "TABULAR",
|
| 134 |
+
"reportBooleanFilter": None,
|
| 135 |
+
"reportFilters": [],
|
| 136 |
+
"detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.Usage_Hours__c"],
|
| 137 |
+
"folderId": LABOPS_REPORTS_FOLDER_ID
|
| 138 |
+
}
|
| 139 |
+
},
|
| 140 |
+
{
|
| 141 |
+
"reportMetadata": {
|
| 142 |
+
"name": f"SmartLog_AMC_Reminders_{timestamp}",
|
| 143 |
+
"developerName": f"SmartLog_AMC_Reminders_{timestamp}",
|
| 144 |
+
"reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
|
| 145 |
+
"reportFormat": "TABULAR",
|
| 146 |
+
"reportBooleanFilter": None,
|
| 147 |
+
"reportFilters": [],
|
| 148 |
+
"detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.AMC_Date__c"],
|
| 149 |
+
"folderId": LABOPS_REPORTS_FOLDER_ID
|
| 150 |
+
}
|
| 151 |
+
}
|
| 152 |
+
]
|
| 153 |
+
for report in reports:
|
| 154 |
+
sf.restful('analytics/reports', method='POST', json=report)
|
| 155 |
+
logging.info("Salesforce reports created")
|
| 156 |
+
except Exception as e:
|
| 157 |
+
logging.error(f"Failed to create Salesforce reports: {str(e)}")
|
| 158 |
+
|
| 159 |
+
# Save to Salesforce
|
| 160 |
+
def save_to_salesforce(df, reminders_df):
|
| 161 |
+
if sf is None:
|
| 162 |
+
return
|
| 163 |
+
try:
|
| 164 |
+
current_date = datetime.now()
|
| 165 |
+
next_30_days = current_date + timedelta(days=30)
|
| 166 |
+
records = []
|
| 167 |
+
reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
|
| 168 |
+
|
| 169 |
+
for _, row in df.iterrows():
|
| 170 |
+
status = str(row['status'])
|
| 171 |
+
log_type = str(row['log_type'])
|
| 172 |
+
status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
|
| 173 |
+
log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
|
| 174 |
+
if status is None or log_type is None:
|
| 175 |
+
continue
|
| 176 |
+
|
| 177 |
+
amc_date_str = None
|
| 178 |
+
if pd.notna(row['amc_date']):
|
| 179 |
+
try:
|
| 180 |
+
amc_date = pd.to_datetime(row['amc_date']).strftime('%Y-%m-%d')
|
| 181 |
+
amc_date_dt = datetime.strptime(amc_date, '%Y-%m-%d')
|
| 182 |
+
if status == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
|
| 183 |
+
logging.info(f"AMC Reminder for Device ID {row['device_id']}")
|
| 184 |
+
except:
|
| 185 |
+
amc_date_str = None
|
| 186 |
+
|
| 187 |
+
record = {
|
| 188 |
+
'Device_Id__c': str(row['device_id'])[:50],
|
| 189 |
+
'Log_Type__c': log_type,
|
| 190 |
+
'Status__c': status,
|
| 191 |
+
'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
|
| 192 |
+
'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
|
| 193 |
+
'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
|
| 194 |
+
'AMC_Date__c': amc_date_str
|
| 195 |
+
}
|
| 196 |
+
if row['device_id'] not in reminder_device_ids:
|
| 197 |
+
records.append(record)
|
| 198 |
+
|
| 199 |
+
if records:
|
| 200 |
+
sf.bulk.SmartLog__c.insert(records)
|
| 201 |
+
logging.info(f"Saved {len(records)} records to Salesforce")
|
| 202 |
+
except Exception as e:
|
| 203 |
+
logging.error(f"Failed to save to Salesforce: {str(e)}")
|
| 204 |
+
|
| 205 |
+
# Summarize logs
|
| 206 |
+
def summarize_logs(df):
|
| 207 |
try:
|
| 208 |
total_devices = df["device_id"].nunique()
|
| 209 |
most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
|
| 210 |
+
prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
|
| 211 |
+
summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
|
| 212 |
+
return summary
|
|
|
|
| 213 |
except Exception as e:
|
| 214 |
+
logging.error(f"Summary generation failed: {str(e)}")
|
| 215 |
+
return f"Failed to generate summary: {str(e)}"
|
| 216 |
|
| 217 |
# Anomaly detection
|
| 218 |
def detect_anomalies(df):
|
| 219 |
try:
|
| 220 |
if "usage_hours" not in df.columns or "downtime" not in df.columns:
|
| 221 |
return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
|
|
|
|
|
|
|
| 222 |
features = df[["usage_hours", "downtime"]].fillna(0)
|
| 223 |
+
if len(features) > 500: # Reduced sample size
|
| 224 |
+
features = features.sample(n=500, random_state=42)
|
| 225 |
+
iso_forest = IsolationForest(contamination=0.1, random_state=42)
|
| 226 |
df["anomaly"] = iso_forest.fit_predict(features)
|
| 227 |
anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
|
| 228 |
if anomalies.empty:
|
| 229 |
return "No anomalies detected.", anomalies
|
| 230 |
+
return "\n".join([f"- Device ID: {row['device_id']}, Usage: {row['usage_hours']}, Downtime: {row['downtime']}, Timestamp: {row['timestamp']}" for _, row in anomalies.head(5).iterrows()]), anomalies
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
except Exception as e:
|
| 232 |
logging.error(f"Anomaly detection failed: {str(e)}")
|
| 233 |
return f"Anomaly detection failed: {str(e)}", pd.DataFrame()
|
|
|
|
| 235 |
# AMC reminders
|
| 236 |
def check_amc_reminders(df, current_date):
|
| 237 |
try:
|
|
|
|
| 238 |
if "device_id" not in df.columns or "amc_date" not in df.columns:
|
|
|
|
| 239 |
return "AMC reminders require 'device_id' and 'amc_date' columns.", pd.DataFrame()
|
|
|
|
| 240 |
df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
|
| 241 |
+
current_date = pd.to_datetime(current_date)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
|
|
|
|
|
|
|
| 243 |
reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
|
| 244 |
if reminders.empty:
|
|
|
|
| 245 |
return "No AMC reminders due within the next 30 days.", reminders
|
| 246 |
+
return "\n".join([f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()]), reminders
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
except Exception as e:
|
| 248 |
logging.error(f"AMC reminder generation failed: {str(e)}")
|
| 249 |
return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
|
| 250 |
|
| 251 |
+
# Dashboard insights
|
| 252 |
+
def generate_dashboard_insights(df):
|
| 253 |
+
try:
|
| 254 |
+
total_devices = df["device_id"].nunique()
|
| 255 |
+
avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
|
| 256 |
+
prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
|
| 257 |
+
insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
|
| 258 |
+
return insights
|
| 259 |
+
except Exception as e:
|
| 260 |
+
logging.error(f"Dashboard insights generation failed: {str(e)}")
|
| 261 |
+
return f"Dashboard insights generation failed: {str(e)}"
|
| 262 |
+
|
| 263 |
# Create usage chart
|
| 264 |
+
def create_usage_chart(df):
|
| 265 |
try:
|
| 266 |
+
if df.empty:
|
|
|
|
|
|
|
|
|
|
| 267 |
return None
|
| 268 |
+
usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
|
| 269 |
if len(usage_data) > 5:
|
| 270 |
usage_data = usage_data.nlargest(5, "usage_hours")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
fig = px.bar(
|
| 272 |
usage_data,
|
| 273 |
x="device_id",
|
| 274 |
y="usage_hours",
|
| 275 |
+
title="Usage Hours per Device",
|
| 276 |
+
labels={"device_id": "Device ID", "usage_hours": "Usage Hours"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
)
|
| 278 |
+
fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
|
| 279 |
return fig
|
| 280 |
except Exception as e:
|
| 281 |
logging.error(f"Failed to create usage chart: {str(e)}")
|
| 282 |
return None
|
| 283 |
|
| 284 |
+
# Create downtime chart
|
| 285 |
+
def create_downtime_chart(df):
|
| 286 |
try:
|
| 287 |
+
downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
if len(downtime_data) > 5:
|
| 289 |
downtime_data = downtime_data.nlargest(5, "downtime")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
fig = px.bar(
|
| 291 |
downtime_data,
|
| 292 |
x="device_id",
|
| 293 |
y="downtime",
|
| 294 |
+
title="Downtime per Device",
|
| 295 |
+
labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 296 |
)
|
| 297 |
+
fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
|
| 298 |
return fig
|
| 299 |
except Exception as e:
|
| 300 |
logging.error(f"Failed to create downtime chart: {str(e)}")
|
| 301 |
return None
|
| 302 |
|
| 303 |
+
# Create daily log trends chart
|
| 304 |
def create_daily_log_trends_chart(df):
|
| 305 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 306 |
df['date'] = df['timestamp'].dt.date
|
| 307 |
+
daily_logs = df.groupby('date').size().reset_index(name='log_count')
|
| 308 |
+
fig = px.line(
|
| 309 |
+
daily_logs,
|
|
|
|
| 310 |
x='date',
|
| 311 |
y='log_count',
|
| 312 |
title="Daily Log Trends",
|
| 313 |
labels={"date": "Date", "log_count": "Number of Logs"}
|
| 314 |
)
|
| 315 |
+
fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
return fig
|
| 317 |
except Exception as e:
|
| 318 |
+
logging.error(f"Failed to create daily log trends chart: {str(e)}")
|
| 319 |
return None
|
| 320 |
|
| 321 |
+
# Create weekly uptime chart
|
| 322 |
def create_weekly_uptime_chart(df):
|
| 323 |
try:
|
| 324 |
+
df['week'] = df['timestamp'].dt.isocalendar().week
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
df['year'] = df['timestamp'].dt.year
|
| 326 |
weekly_data = df.groupby(['year', 'week']).agg({
|
| 327 |
+
'usage_hours': 'sum',
|
| 328 |
'downtime': 'sum'
|
| 329 |
}).reset_index()
|
| 330 |
+
weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
|
| 331 |
+
weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 332 |
fig = px.bar(
|
| 333 |
weekly_data,
|
| 334 |
+
x='year_week',
|
| 335 |
+
y='uptime_percent',
|
| 336 |
title="Weekly Uptime Percentage",
|
| 337 |
+
labels={"year_week": "Year-Week", "uptime_percent": "Uptime %"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
)
|
| 339 |
+
fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
|
| 340 |
return fig
|
| 341 |
except Exception as e:
|
| 342 |
+
logging.error(f"Failed to create weekly uptime chart: {str(e)}")
|
| 343 |
return None
|
| 344 |
|
| 345 |
+
# Create anomaly alerts chart
|
| 346 |
+
def create_anomaly_alerts_chart(anomalies_df):
|
| 347 |
try:
|
| 348 |
+
if anomalies_df.empty:
|
|
|
|
| 349 |
return None
|
| 350 |
+
anomalies_df['date'] = anomalies_df['timestamp'].dt.date
|
| 351 |
+
anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
|
|
|
|
|
|
|
|
|
|
| 352 |
fig = px.scatter(
|
| 353 |
+
anomaly_counts,
|
| 354 |
+
x='date',
|
| 355 |
+
y='anomaly_count',
|
| 356 |
+
title="Anomaly Alerts Over Time",
|
| 357 |
+
labels={"date": "Date", "anomaly_count": "Number of Anomalies"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
)
|
| 359 |
+
fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
|
| 360 |
return fig
|
| 361 |
except Exception as e:
|
| 362 |
+
logging.error(f"Failed to create anomaly alerts chart: {str(e)}")
|
| 363 |
return None
|
| 364 |
|
| 365 |
+
# Generate device cards
|
| 366 |
def generate_device_cards(df):
|
| 367 |
try:
|
| 368 |
if df.empty:
|
|
|
|
| 369 |
return '<p>No devices available to display.</p>'
|
| 370 |
+
device_stats = df.groupby('device_id').agg({
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
'status': 'last',
|
| 372 |
'timestamp': 'max',
|
| 373 |
}).reset_index()
|
| 374 |
+
device_stats['count'] = df.groupby('device_id').size().reindex(device_stats['device_id']).values
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 375 |
device_stats['health'] = device_stats['status'].map({
|
| 376 |
'Active': 'Healthy',
|
| 377 |
'Inactive': 'Unhealthy',
|
| 378 |
'Pending': 'Warning'
|
| 379 |
}).fillna('Unknown')
|
|
|
|
| 380 |
cards_html = '<div style="display: flex; flex-wrap: wrap; gap: 20px;">'
|
| 381 |
for _, row in device_stats.iterrows():
|
| 382 |
+
health_color = {'Healthy': 'green', 'Unhealthy': 'red', 'Warning': 'orange', 'Unknown': 'gray'}.get(row['health'], 'gray')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
timestamp_str = str(row['timestamp']) if pd.notna(row['timestamp']) else 'Unknown'
|
| 384 |
+
cards_html += f"""
|
| 385 |
<div style="border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; width: 200px;">
|
| 386 |
<h4>Device: {row['device_id']}</h4>
|
| 387 |
<p><b>Health:</b> <span style="color: {health_color}">{row['health']}</span></p>
|
|
|
|
| 389 |
<p><b>Last Log:</b> {timestamp_str}</p>
|
| 390 |
</div>
|
| 391 |
"""
|
|
|
|
| 392 |
cards_html += '</div>'
|
|
|
|
| 393 |
return cards_html
|
| 394 |
except Exception as e:
|
| 395 |
+
logging.error(f"Failed to generate device cards: {str(e)}")
|
| 396 |
return f'<p>Error generating device cards: {str(e)}</p>'
|
| 397 |
|
| 398 |
+
# Generate monthly status
|
| 399 |
def generate_monthly_status(df, selected_month):
|
| 400 |
try:
|
| 401 |
total_devices = df['device_id'].nunique()
|
| 402 |
total_usage_hours = df['usage_hours'].sum()
|
| 403 |
total_downtime = df['downtime'].sum()
|
| 404 |
+
avg_usage = total_usage_hours / total_devices if total_devices > 0 else 0
|
| 405 |
+
avg_downtime = total_downtime / total_devices if total_devices > 0 else 0
|
| 406 |
+
return f"""
|
|
|
|
| 407 |
Monthly Status for {selected_month}:
|
| 408 |
- Total Devices: {total_devices}
|
| 409 |
- Total Usage Hours: {total_usage_hours:.2f}
|
| 410 |
- Total Downtime Hours: {total_downtime:.2f}
|
| 411 |
+
- Average Usage per Device: {avg_usage:.2f} hours
|
| 412 |
+
- Average Downtime per Device: {avg_downtime:.2f} hours
|
| 413 |
"""
|
|
|
|
| 414 |
except Exception as e:
|
| 415 |
logging.error(f"Failed to generate monthly status: {str(e)}")
|
| 416 |
return f"Failed to generate monthly status: {str(e)}"
|
| 417 |
|
| 418 |
# Generate PDF content
|
| 419 |
+
def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
|
| 420 |
if not reportlab_available:
|
|
|
|
| 421 |
return None
|
| 422 |
try:
|
|
|
|
| 423 |
pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
|
| 424 |
doc = SimpleDocTemplate(pdf_path, pagesize=letter)
|
| 425 |
styles = getSampleStyleSheet()
|
| 426 |
story = []
|
| 427 |
|
| 428 |
def safe_paragraph(text, style):
|
| 429 |
+
return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
|
|
|
|
| 430 |
|
| 431 |
story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
|
| 432 |
story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
|
| 433 |
story.append(Spacer(1, 12))
|
| 434 |
|
| 435 |
+
if selected_month != "All":
|
| 436 |
monthly_status = generate_monthly_status(df, selected_month)
|
| 437 |
story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
|
| 438 |
story.append(safe_paragraph(monthly_status, styles['Normal']))
|
|
|
|
| 443 |
story.append(Spacer(1, 12))
|
| 444 |
|
| 445 |
story.append(Paragraph("Log Preview", styles['Heading2']))
|
| 446 |
+
if not preview_df.empty:
|
| 447 |
+
data = [preview_df.columns.tolist()] + preview_df.head(5).values.tolist()
|
| 448 |
+
table = Table(data)
|
| 449 |
+
table.setStyle(TableStyle([
|
| 450 |
+
('BACKGROUND', (0, 0), (-1, 0), colors.grey),
|
| 451 |
+
('TEXTCOLOR', (0, 0), (-1, 0), colors.whitesmoke),
|
| 452 |
+
('ALIGN', (0, 0), (-1, -1), 'CENTER'),
|
| 453 |
+
('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
|
| 454 |
+
('FONTSIZE', (0, 0), (-1, 0), 12),
|
| 455 |
+
('BOTTOMPADDING', (0, 0), (-1, 0), 12),
|
| 456 |
+
('BACKGROUND', (0, 1), (-1, -1), colors.beige),
|
| 457 |
+
('TEXTCOLOR', (0, 1), (-1, -1), colors.black),
|
| 458 |
+
('FONTNAME', (0, 1), (-1, -1), 'Helvetica'),
|
| 459 |
+
('FONTSIZE', (0, 1), (-1, -1), 10),
|
| 460 |
+
('GRID', (0, 0), (-1, -1), 1, colors.black)
|
| 461 |
+
]))
|
| 462 |
+
story.append(table)
|
| 463 |
+
else:
|
| 464 |
+
story.append(safe_paragraph("No preview available.", styles['Normal']))
|
| 465 |
story.append(Spacer(1, 12))
|
| 466 |
|
| 467 |
story.append(Paragraph("Device Cards", styles['Heading2']))
|
| 468 |
+
device_cards_text = device_cards_html.replace('<div>', '').replace('</div>', '\n').replace('<h4>', '').replace('</h4>', '\n').replace('<p>', '').replace('</p>', '\n').replace('<b>', '').replace('</b>', '').replace('<span style="color: green">', '').replace('<span style="color: red">', '').replace('<span style="color: orange">', '').replace('<span style="color: gray">', '').replace('</span>', '')
|
| 469 |
story.append(safe_paragraph(device_cards_text, styles['Normal']))
|
| 470 |
story.append(Spacer(1, 12))
|
| 471 |
|
|
|
|
| 481 |
story.append(safe_paragraph(insights, styles['Normal']))
|
| 482 |
story.append(Spacer(1, 12))
|
| 483 |
|
| 484 |
+
story.append(Paragraph("Charts", styles['Heading2']))
|
| 485 |
+
story.append(Paragraph("[Chart placeholders - see dashboard for visuals]", styles['Normal']))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 486 |
|
| 487 |
doc.build(story)
|
| 488 |
logging.info(f"PDF generated at {pdf_path}")
|
| 489 |
return pdf_path
|
| 490 |
except Exception as e:
|
| 491 |
+
logging.error(f"Failed to generate PDF: {str(e)}")
|
| 492 |
return None
|
| 493 |
|
| 494 |
+
# Main processing function
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 495 |
async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
|
| 496 |
+
start_time = time.time()
|
| 497 |
try:
|
|
|
|
|
|
|
| 498 |
if not file_obj:
|
| 499 |
+
return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
|
| 500 |
+
|
| 501 |
file_path = file_obj.name
|
| 502 |
current_modified_time = os.path.getmtime(file_path)
|
|
|
|
| 503 |
if last_modified_state and current_modified_time == last_modified_state:
|
| 504 |
+
return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state
|
| 505 |
|
| 506 |
+
logging.info(f"Processing file: {file_path}")
|
|
|
|
| 507 |
if not file_path.endswith(".csv"):
|
| 508 |
+
return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
|
| 509 |
|
| 510 |
required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
|
| 511 |
dtypes = {
|
|
|
|
| 517 |
"amc_date": "string"
|
| 518 |
}
|
| 519 |
df = pd.read_csv(file_path, dtype=dtypes)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
missing_columns = [col for col in required_columns if col not in df.columns]
|
| 521 |
if missing_columns:
|
| 522 |
+
return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
|
| 523 |
+
|
| 524 |
df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
|
| 525 |
df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
|
| 526 |
if df["timestamp"].dt.tz is None:
|
|
|
|
| 527 |
df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
|
| 528 |
if df.empty:
|
| 529 |
+
return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 530 |
|
| 531 |
+
# Apply filters
|
| 532 |
+
filtered_df = df.copy()
|
| 533 |
if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
|
| 534 |
filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
|
|
|
|
|
|
|
| 535 |
if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
|
| 536 |
filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
|
|
|
|
|
|
|
| 537 |
if date_range and len(date_range) == 2:
|
| 538 |
days_start, days_end = date_range
|
| 539 |
today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
|
| 540 |
start_date = today + pd.Timedelta(days=days_start)
|
| 541 |
+
end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
|
|
|
|
|
|
|
| 542 |
filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
|
|
|
|
|
|
|
| 543 |
if month_filter and month_filter != "All":
|
| 544 |
selected_date = pd.to_datetime(month_filter, format="%B %Y")
|
| 545 |
filtered_df = filtered_df[
|
| 546 |
(filtered_df['timestamp'].dt.year == selected_date.year) &
|
| 547 |
(filtered_df['timestamp'].dt.month == selected_date.month)
|
| 548 |
]
|
|
|
|
| 549 |
|
| 550 |
if filtered_df.empty:
|
| 551 |
+
return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
|
|
|
|
|
|
|
|
|
|
| 552 |
|
| 553 |
+
# Generate table for preview
|
| 554 |
+
preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
|
| 555 |
+
preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
|
| 557 |
# Run tasks concurrently
|
| 558 |
+
with ThreadPoolExecutor(max_workers=6) as executor:
|
| 559 |
+
future_summary = executor.submit(summarize_logs, filtered_df)
|
| 560 |
future_anomalies = executor.submit(detect_anomalies, filtered_df)
|
| 561 |
future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
|
| 562 |
+
future_insights = executor.submit(generate_dashboard_insights, filtered_df)
|
| 563 |
+
future_usage_chart = executor.submit(create_usage_chart, filtered_df)
|
| 564 |
+
future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
|
| 565 |
future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
|
| 566 |
future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
|
| 567 |
+
future_anomaly_alerts_chart = executor.submit(create_anomaly_alerts_chart, pd.DataFrame())
|
| 568 |
future_device_cards = executor.submit(generate_device_cards, filtered_df)
|
| 569 |
+
future_reports = executor.submit(create_salesforce_reports, filtered_df)
|
| 570 |
|
| 571 |
+
summary = f"Step 1: Summary Report\n{future_summary.result()}"
|
|
|
|
|
|
|
| 572 |
anomalies, anomalies_df = future_anomalies.result()
|
| 573 |
anomalies = f"Anomaly Detection\n{anomalies}"
|
| 574 |
amc_reminders, reminders_df = future_amc.result()
|
| 575 |
amc_reminders = f"AMC Reminders\n{amc_reminders}"
|
| 576 |
+
insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
|
| 577 |
usage_chart = future_usage_chart.result()
|
| 578 |
downtime_chart = future_downtime_chart.result()
|
| 579 |
daily_log_chart = future_daily_log_chart.result()
|
| 580 |
weekly_uptime_chart = future_weekly_uptime_chart.result()
|
| 581 |
+
anomaly_alerts_chart = future_anomaly_alerts_chart.result()
|
| 582 |
device_cards = future_device_cards.result()
|
| 583 |
|
| 584 |
+
save_to_salesforce(filtered_df, reminders_df)
|
| 585 |
+
pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, filtered_df, month_filter)
|
| 586 |
|
| 587 |
+
elapsed_time = time.time() - start_time
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
|
| 589 |
if elapsed_time > 10:
|
| 590 |
logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
|
| 591 |
|
| 592 |
+
return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
|
| 593 |
except Exception as e:
|
| 594 |
logging.error(f"Failed to process file: {str(e)}")
|
| 595 |
+
return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state
|
| 596 |
|
| 597 |
+
# Update filters
|
| 598 |
def update_filters(file_obj):
|
| 599 |
if not file_obj:
|
|
|
|
| 600 |
return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
|
|
|
|
| 601 |
try:
|
|
|
|
| 602 |
with open(file_obj.name, 'rb') as f:
|
| 603 |
csv_content = f.read().decode('utf-8')
|
| 604 |
df = pd.read_csv(io.StringIO(csv_content))
|
| 605 |
df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
|
| 606 |
+
|
| 607 |
+
lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
|
| 608 |
+
equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
|
| 609 |
+
month_options = ['All'] + sorted(df['timestamp'].dt.strftime('%B %Y').dropna().unique().tolist()) if 'timestamp' in df.columns else ['All']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 610 |
|
| 611 |
return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
|
| 612 |
except Exception as e:
|
|
|
|
| 623 |
.dashboard-section h3 {font-size: 18px; margin-bottom: 2px;}
|
| 624 |
.dashboard-section p {margin: 1px 0; line-height: 1.2;}
|
| 625 |
.dashboard-section ul {margin: 2px 0; padding-left: 20px;}
|
| 626 |
+
.table {width: 100%; border-collapse: collapse;}
|
| 627 |
+
.table th, .table td {border: 1px solid #ddd; padding: 8px; text-align: left;}
|
| 628 |
+
.table th {background-color: #f2f2f2;}
|
| 629 |
+
.table tr:nth-child(even) {background-color: #f9f9f9;}
|
| 630 |
""") as iface:
|
| 631 |
+
gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
|
| 632 |
+
gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data.")
|
| 633 |
|
| 634 |
last_modified_state = gr.State(value=None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
|
| 636 |
with gr.Row():
|
| 637 |
with gr.Column(scale=1):
|
| 638 |
file_input = gr.File(label="Upload Logs (CSV)", file_types=[".csv"])
|
|
|
|
| 639 |
with gr.Group():
|
| 640 |
gr.Markdown("### Filters")
|
| 641 |
+
lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
|
| 642 |
+
equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
|
| 643 |
+
date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
|
| 644 |
+
month_filter = gr.Dropdown(label="Select Month for Report", choices=['All'], value='All', interactive=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 645 |
submit_button = gr.Button("Analyze", variant="primary")
|
| 646 |
|
| 647 |
with gr.Column(scale=2):
|
| 648 |
with gr.Group(elem_classes="dashboard-container"):
|
| 649 |
gr.Markdown("<div class='dashboard-title'>Analysis Results</div>")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 650 |
with gr.Group(elem_classes="dashboard-section"):
|
| 651 |
gr.Markdown("### Step 1: Summary Report")
|
| 652 |
summary_output = gr.Markdown()
|
|
|
|
| 653 |
with gr.Group(elem_classes="dashboard-section"):
|
| 654 |
gr.Markdown("### Step 2: Log Preview")
|
| 655 |
preview_output = gr.HTML()
|
|
|
|
| 656 |
with gr.Group(elem_classes="dashboard-section"):
|
| 657 |
gr.Markdown("### Device Cards")
|
| 658 |
device_cards_output = gr.HTML()
|
|
|
|
| 659 |
with gr.Group(elem_classes="dashboard-section"):
|
| 660 |
gr.Markdown("### Charts")
|
| 661 |
with gr.Tab("Usage Hours per Device"):
|
|
|
|
| 668 |
weekly_uptime_output = gr.Plot()
|
| 669 |
with gr.Tab("Anomaly Alerts"):
|
| 670 |
anomaly_alerts_output = gr.Plot()
|
|
|
|
| 671 |
with gr.Group(elem_classes="dashboard-section"):
|
| 672 |
gr.Markdown("### Step 4: Anomaly Detection")
|
| 673 |
anomaly_output = gr.Markdown()
|
|
|
|
| 674 |
with gr.Group(elem_classes="dashboard-section"):
|
| 675 |
gr.Markdown("### Step 5: AMC Reminders")
|
| 676 |
amc_output = gr.Markdown()
|
|
|
|
| 677 |
with gr.Group(elem_classes="dashboard-section"):
|
| 678 |
gr.Markdown("### Step 6: Insights (AI)")
|
| 679 |
insights_output = gr.Markdown()
|
|
|
|
| 680 |
with gr.Group(elem_classes="dashboard-section"):
|
| 681 |
gr.Markdown("### Export Report")
|
| 682 |
pdf_output = gr.File(label="Download Monthly Status Report as PDF")
|
|
|
|
| 690 |
|
| 691 |
submit_button.click(
|
| 692 |
fn=process_logs,
|
| 693 |
+
inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, month_filter, last_modified_state],
|
| 694 |
+
outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_chart, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 695 |
)
|
| 696 |
|
| 697 |
logging.info("Gradio interface initialized successfully")
|