RathodHarish commited on
Commit
dc56f41
·
verified ·
1 Parent(s): d0b0f06

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +333 -671
app.py CHANGED
@@ -1,67 +1,233 @@
1
- """
2
- LabOps Log Analyzer Dashboard with CSV file upload, PDF generation, and email alerts
3
- """
4
  import gradio as gr
5
  import pandas as pd
6
  from datetime import datetime, timedelta
7
  import logging
8
  import plotly.express as px
9
  from sklearn.ensemble import IsolationForest
 
 
10
  from concurrent.futures import ThreadPoolExecutor
 
11
  import os
 
12
  import io
13
- import smtplib
14
- from email.mime.text import MIMEText
15
- from email.mime.multipart import MIMEMultipart
16
- from email.mime.application import MIMEApplication
17
 
18
  # Configure logging
19
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  # Try to import reportlab
22
  try:
23
  from reportlab.lib.pagesizes import letter
24
- from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
25
  from reportlab.lib.styles import getSampleStyleSheet
 
26
  reportlab_available = True
27
  logging.info("reportlab module successfully imported")
28
  except ImportError:
29
  logging.warning("reportlab module not found. PDF generation disabled.")
30
  reportlab_available = False
31
 
32
- # Generate summary and insights without Hugging Face model
33
- def generate_summary_and_insights(df):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  try:
35
  total_devices = df["device_id"].nunique()
36
  most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
37
- avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
38
- summary = f"Maintenance logs for {total_devices} devices. Most used: {most_used}."
39
- insights = f"{total_devices} devices, average usage {avg_usage:.2f} hours."
40
- return summary, f"Insights: {insights}"
41
  except Exception as e:
42
- logging.error(f"Summary and insights generation failed: {str(e)}")
43
- return f"Failed to generate summary: {str(e)}", f"Failed to generate insights: {str(e)}"
44
 
45
  # Anomaly detection
46
  def detect_anomalies(df):
47
  try:
48
  if "usage_hours" not in df.columns or "downtime" not in df.columns:
49
  return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
50
- if len(df) > 1000:
51
- df = df.sample(n=1000, random_state=42)
52
  features = df[["usage_hours", "downtime"]].fillna(0)
53
- iso_forest = IsolationForest(contamination=0.1, random_state=42, n_jobs=-1)
 
 
54
  df["anomaly"] = iso_forest.fit_predict(features)
55
  anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
56
  if anomalies.empty:
57
  return "No anomalies detected.", anomalies
58
- anomaly_lines = ["Detected Anomalies:"]
59
- for _, row in anomalies.head(5).iterrows():
60
- anomaly_lines.append(
61
- f"- Device ID: {row['device_id']}, Usage Hours: {row['usage_hours']}, "
62
- f"Downtime: {row['downtime']}, Timestamp: {row['timestamp']}"
63
- )
64
- return "\n".join(anomaly_lines), anomalies
65
  except Exception as e:
66
  logging.error(f"Anomaly detection failed: {str(e)}")
67
  return f"Anomaly detection failed: {str(e)}", pd.DataFrame()
@@ -69,387 +235,153 @@ def detect_anomalies(df):
69
  # AMC reminders
70
  def check_amc_reminders(df, current_date):
71
  try:
72
- logging.info(f"Input DataFrame for AMC reminders:\n{df.head().to_string()}")
73
  if "device_id" not in df.columns or "amc_date" not in df.columns:
74
- logging.warning("Missing 'device_id' or 'amc_date' columns for AMC reminders.")
75
  return "AMC reminders require 'device_id' and 'amc_date' columns.", pd.DataFrame()
76
-
77
  df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
78
- if df["amc_date"].dt.tz is None:
79
- logging.info("Localizing naive AMC dates to IST")
80
- df["amc_date"] = df["amc_date"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
81
-
82
- current_date = pd.to_datetime(current_date).tz_localize('Asia/Kolkata')
83
- logging.info(f"Current date for AMC check: {current_date}")
84
-
85
  df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
86
- logging.info(f"Days to AMC:\n{df[['device_id', 'amc_date', 'days_to_amc']].to_string()}")
87
-
88
  reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
89
  if reminders.empty:
90
- logging.info("No AMC reminders found within the next 30 days.")
91
  return "No AMC reminders due within the next 30 days.", reminders
92
-
93
- reminder_lines = ["Upcoming AMC Reminders:"]
94
- for _, row in reminders.head(5).iterrows():
95
- reminder_lines.append(f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}")
96
- logging.info(f"Found {len(reminders)} AMC reminders: {reminder_lines}")
97
- return "\n".join(reminder_lines), reminders
98
  except Exception as e:
99
  logging.error(f"AMC reminder generation failed: {str(e)}")
100
  return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
101
 
 
 
 
 
 
 
 
 
 
 
 
 
102
  # Create usage chart
103
- def create_usage_chart(agg_data):
104
  try:
105
- usage_data = agg_data['usage_per_device']
106
- logging.info(f"Usage data for chart: {usage_data.to_string()}")
107
- if usage_data.empty:
108
- logging.warning("Usage data is empty.")
109
  return None
 
110
  if len(usage_data) > 5:
111
  usage_data = usage_data.nlargest(5, "usage_hours")
112
-
113
- q75, q25 = usage_data["usage_hours"].quantile([0.75, 0.25])
114
- iqr = q75 - q25
115
- spike_threshold = q75 + 1.5 * iqr
116
- usage_data["color"] = usage_data["usage_hours"].apply(
117
- lambda x: "red" if x > spike_threshold else "teal"
118
- )
119
-
120
  fig = px.bar(
121
  usage_data,
122
  x="device_id",
123
  y="usage_hours",
124
- title="Usage Hours per Device (Red = Usage Spike)",
125
- labels={"device_id": "Device ID", "usage_hours": "Usage Hours"},
126
- color="color",
127
- color_discrete_map={"teal": "#4ECDC4", "red": "#FF0000"}
128
- )
129
- fig.update_traces(
130
- marker_line_color='#333333',
131
- marker_line_width=1.5,
132
- opacity=0.9
133
- )
134
- fig.update_layout(
135
- title_font=dict(size=18, family="Arial", color="#333333"),
136
- font=dict(family="Arial", size=12, color="#333333"),
137
- plot_bgcolor="white",
138
- paper_bgcolor="white",
139
- margin=dict(l=30, r=30, t=50, b=30),
140
- xaxis=dict(
141
- title="Device ID",
142
- showgrid=False,
143
- tickangle=45,
144
- title_font=dict(size=14),
145
- tickfont=dict(size=12)
146
- ),
147
- yaxis=dict(
148
- title="Usage Hours",
149
- gridcolor="#E5E5E5",
150
- gridwidth=1,
151
- title_font=dict(size=14),
152
- tickfont=dict(size=12)
153
- ),
154
- showlegend=False,
155
- bargap=0.2
156
  )
 
157
  return fig
158
  except Exception as e:
159
  logging.error(f"Failed to create usage chart: {str(e)}")
160
  return None
161
 
162
- # Create downtime chart (fixed syntax error)
163
- def create_downtime_chart(agg_data):
164
  try:
165
- downtime_data = agg_data['downtime_per_device']
166
- logging.info(f"Downtime data for chart: {downtime_data.to_string()}")
167
- if downtime_data.empty:
168
- logging.warning("Downtime data is empty.")
169
- return None
170
  if len(downtime_data) > 5:
171
  downtime_data = downtime_data.nlargest(5, "downtime")
172
-
173
- q75, q25 = downtime_data["downtime"].quantile([0.75, 0.25])
174
- iqr = q75 - q25
175
- spike_threshold = q75 + 1.5 * iqr
176
- downtime_data["color"] = downtime_data["downtime"].apply(
177
- lambda x: "red" if x > spike_threshold else "green"
178
- )
179
-
180
  fig = px.bar(
181
  downtime_data,
182
  x="device_id",
183
  y="downtime",
184
- title="Downtime per Device (Red = Downtime Spike)",
185
- labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"},
186
- color="color",
187
- color_discrete_map={"green": "#96CEB4", "red": "#FF0000"}
188
- )
189
- fig.update_traces(
190
- marker_line_color='#333333',
191
- marker_line_width=1.5,
192
- opacity=0.9
193
- )
194
- fig.update_layout(
195
- title_font=dict(size=18, family="Arial", color="#333333"),
196
- font=dict(family="Arial", size=12, color="#333333"),
197
- plot_bgcolor="white",
198
- paper_bgcolor="white",
199
- margin=dict(l=30, r=30, t=50, b=30),
200
- xaxis=dict(
201
- title="Device ID",
202
- showgrid=False,
203
- tickangle=45,
204
- title_font=dict(size=14),
205
- tickfont=dict(size=12)
206
- ),
207
- yaxis=dict(
208
- title="Downtime (Hours)",
209
- gridcolor="#E5E5E5",
210
- gridwidth=1,
211
- title_font=dict(size=14),
212
- tickfont=dict(size=12)
213
- ),
214
- showlegend=False,
215
- bargap=0.2
216
  )
 
217
  return fig
218
  except Exception as e:
219
  logging.error(f"Failed to create downtime chart: {str(e)}")
220
  return None
221
 
222
- # Create Daily Log Trends chart (area chart with markers)
223
  def create_daily_log_trends_chart(df):
224
  try:
225
- if df.empty or 'timestamp' not in df.columns:
226
- logging.warning("DataFrame is empty or missing 'timestamp' column for Daily Log Trends.")
227
- return None
228
-
229
- # Group by date to count logs per day
230
  df['date'] = df['timestamp'].dt.date
231
- log_counts = df.groupby('date').size().reset_index(name='log_count')
232
-
233
- fig = px.area(
234
- log_counts,
235
  x='date',
236
  y='log_count',
237
  title="Daily Log Trends",
238
  labels={"date": "Date", "log_count": "Number of Logs"}
239
  )
240
- # Add markers
241
- fig.update_traces(
242
- fill='tozeroy',
243
- line_color='#4ECDC4',
244
- line_width=2,
245
- mode='lines+markers',
246
- marker=dict(size=8, color='#4ECDC4', line=dict(width=1, color='#333333')),
247
- fillcolor='rgba(78, 205, 196, 0.3)' # Gradient fill with transparency
248
- )
249
- fig.update_layout(
250
- title_font=dict(size=18, family="Arial", color="#333333"),
251
- font=dict(family="Arial", size=12, color="#333333"),
252
- plot_bgcolor="white",
253
- paper_bgcolor="white",
254
- margin=dict(l=30, r=30, t=50, b=30),
255
- xaxis=dict(
256
- title="Date",
257
- showgrid=False,
258
- title_font=dict(size=14),
259
- tickfont=dict(size=12)
260
- ),
261
- yaxis=dict(
262
- title="Number of Logs",
263
- gridcolor="#E5E5E5",
264
- gridwidth=1,
265
- title_font=dict(size=14),
266
- tickfont=dict(size=12)
267
- )
268
- )
269
  return fig
270
  except Exception as e:
271
- logging.error(f"Failed to create Daily Log Trends chart: {str(e)}")
272
  return None
273
 
274
- # Create Weekly Uptime Percentage chart
275
  def create_weekly_uptime_chart(df):
276
  try:
277
- if df.empty or 'timestamp' not in df.columns or 'downtime' not in df.columns:
278
- logging.warning("DataFrame is empty or missing required columns for Weekly Uptime Percentage.")
279
- return None
280
-
281
- logging.info(f"DataFrame for Weekly Uptime:\n{df[['timestamp', 'downtime']].to_string()}")
282
-
283
- # Group by week (handle pandas 2.x compatibility)
284
- try:
285
- df['week'] = df['timestamp'].dt.isocalendar().week
286
- except AttributeError:
287
- # For pandas 2.x, use .dt.weekofyear or manual calculation
288
- df['week'] = df['timestamp'].dt.isocalendar()['week']
289
  df['year'] = df['timestamp'].dt.year
290
  weekly_data = df.groupby(['year', 'week']).agg({
 
291
  'downtime': 'sum'
292
  }).reset_index()
293
-
294
- logging.info(f"Weekly data:\n{weekly_data.to_string()}")
295
-
296
- # Calculate uptime percentage (assuming 24*7 = 168 hours per week)
297
- total_hours_per_week = 168
298
- weekly_data['uptime_percentage'] = ((total_hours_per_week - weekly_data['downtime']) / total_hours_per_week) * 100
299
- weekly_data['uptime_percentage'] = weekly_data['uptime_percentage'].clip(0, 100) # Ensure percentage is between 0 and 100
300
- weekly_data['week_label'] = weekly_data.apply(lambda x: f"{x['year']}-W{x['week']:02d}", axis=1)
301
-
302
- if weekly_data.empty:
303
- logging.warning("No weekly data available for Weekly Uptime Percentage chart.")
304
- return None
305
-
306
  fig = px.bar(
307
  weekly_data,
308
- x='week_label',
309
- y='uptime_percentage',
310
  title="Weekly Uptime Percentage",
311
- labels={"week_label": "Week", "uptime_percentage": "Uptime Percentage (%)"},
312
- color='uptime_percentage',
313
- color_continuous_scale=['#FF0000', '#96CEB4']
314
- )
315
- fig.update_traces(
316
- marker_line_color='#333333',
317
- marker_line_width=1.5,
318
- opacity=0.9
319
- )
320
- fig.update_layout(
321
- title_font=dict(size=18, family="Arial", color="#333333"),
322
- font=dict(family="Arial", size=12, color="#333333"),
323
- plot_bgcolor="white",
324
- paper_bgcolor="white",
325
- margin=dict(l=30, r=30, t=50, b=30),
326
- xaxis=dict(
327
- title="Week",
328
- showgrid=False,
329
- tickangle=45,
330
- title_font=dict(size=14),
331
- tickfont=dict(size=12)
332
- ),
333
- yaxis=dict(
334
- title="Uptime Percentage (%)",
335
- gridcolor="#E5E5E5",
336
- gridwidth=1,
337
- title_font=dict(size=14),
338
- tickfont=dict(size=12)
339
- ),
340
- showlegend=False,
341
- bargap=0.2
342
  )
 
343
  return fig
344
  except Exception as e:
345
- logging.error(f"Failed to create Weekly Uptime Percentage chart: {str(e)}")
346
  return None
347
 
348
- # Create Anomaly Alerts chart (bubble chart)
349
- def create_anomaly_alerts_chart(df, anomalies_df):
350
  try:
351
- if df.empty or anomalies_df.empty:
352
- logging.warning("DataFrame or anomalies DataFrame is empty for Anomaly Alerts chart.")
353
  return None
354
-
355
- # Prepare data for bubble chart
356
- df['is_anomaly'] = df.index.isin(anomalies_df.index)
357
- df['color'] = df['is_anomaly'].map({True: 'red', False: 'blue'})
358
-
359
  fig = px.scatter(
360
- df,
361
- x='usage_hours',
362
- y='downtime',
363
- size='usage_hours', # Bubble size based on usage hours
364
- color='color',
365
- title="Anomaly Alerts (Red = Anomaly)",
366
- labels={"usage_hours": "Usage Hours", "downtime": "Downtime (Hours)"},
367
- color_discrete_map={'blue': '#4ECDC4', 'red': '#FF0000'}
368
- )
369
- fig.update_traces(
370
- marker=dict(
371
- sizemode='area',
372
- sizeref=0.1, # Adjust bubble size scaling
373
- line=dict(width=1, color='#333333')
374
- ),
375
- opacity=0.7
376
- )
377
- fig.update_layout(
378
- title_font=dict(size=18, family="Arial", color="#333333"),
379
- font=dict(family="Arial", size=12, color="#333333"),
380
- plot_bgcolor="white",
381
- paper_bgcolor="white",
382
- margin=dict(l=30, r=30, t=50, b=30),
383
- xaxis=dict(
384
- title="Usage Hours",
385
- showgrid=False,
386
- title_font=dict(size=14),
387
- tickfont=dict(size=12)
388
- ),
389
- yaxis=dict(
390
- title="Downtime (Hours)",
391
- gridcolor="#E5E5E5",
392
- gridwidth=1,
393
- title_font=dict(size=14),
394
- tickfont=dict(size=12)
395
- ),
396
- showlegend=False
397
  )
 
398
  return fig
399
  except Exception as e:
400
- logging.error(f"Failed to create Anomaly Alerts chart: {str(e)}")
401
  return None
402
 
403
- # Generate Device Cards HTML
404
  def generate_device_cards(df):
405
  try:
406
  if df.empty:
407
- logging.warning("DataFrame is empty in generate_device_cards.")
408
  return '<p>No devices available to display.</p>'
409
-
410
- required_columns = ['device_id', 'status', 'timestamp']
411
- missing_columns = [col for col in required_columns if col not in df.columns]
412
- if missing_columns:
413
- logging.error(f"Missing required columns in DataFrame: {missing_columns}")
414
- return f'<p>Error: Missing required columns: {missing_columns}</p>'
415
-
416
- if df['timestamp'].isna().all():
417
- logging.warning("All timestamps are NaT. Cannot generate device cards.")
418
- return '<p>Error: All timestamps are invalid.</p>'
419
-
420
- df_clean = df.dropna(subset=['timestamp']).copy()
421
- if df_clean.empty:
422
- logging.warning("DataFrame is empty after dropping NaT timestamps.")
423
- return '<p>No valid timestamps available to display.</p>'
424
-
425
- device_stats = df_clean.groupby('device_id').agg({
426
  'status': 'last',
427
  'timestamp': 'max',
428
  }).reset_index()
429
-
430
- counts = df_clean.groupby('device_id').size().reset_index(name='count')
431
- device_stats = device_stats.merge(counts, on='device_id')
432
-
433
- # Limit to top 10 devices by count
434
- device_stats = device_stats.nlargest(10, 'count')
435
- logging.info(f"Limited device cards to top {len(device_stats)} devices by usage count.")
436
-
437
  device_stats['health'] = device_stats['status'].map({
438
  'Active': 'Healthy',
439
  'Inactive': 'Unhealthy',
440
  'Pending': 'Warning'
441
  }).fillna('Unknown')
442
-
443
  cards_html = '<div style="display: flex; flex-wrap: wrap; gap: 20px;">'
444
  for _, row in device_stats.iterrows():
445
- health_color = {
446
- 'Healthy': 'green',
447
- 'Unhealthy': 'red',
448
- 'Warning': 'orange',
449
- 'Unknown': 'gray'
450
- }.get(row['health'], 'gray')
451
  timestamp_str = str(row['timestamp']) if pd.notna(row['timestamp']) else 'Unknown'
452
- card = f"""
453
  <div style="border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; width: 200px;">
454
  <h4>Device: {row['device_id']}</h4>
455
  <p><b>Health:</b> <span style="color: {health_color}">{row['health']}</span></p>
@@ -457,57 +389,50 @@ def generate_device_cards(df):
457
  <p><b>Last Log:</b> {timestamp_str}</p>
458
  </div>
459
  """
460
- cards_html += card
461
  cards_html += '</div>'
462
- logging.info("Device cards generated successfully")
463
  return cards_html
464
  except Exception as e:
465
- logging.error(f"Failed to generate device cards: {str(e)}", exc_info=True)
466
  return f'<p>Error generating device cards: {str(e)}</p>'
467
 
468
- # Generate monthly status summary for PDF
469
  def generate_monthly_status(df, selected_month):
470
  try:
471
  total_devices = df['device_id'].nunique()
472
  total_usage_hours = df['usage_hours'].sum()
473
  total_downtime = df['downtime'].sum()
474
- avg_usage_per_device = total_usage_hours / total_devices if total_devices > 0 else 0
475
- avg_downtime_per_device = total_downtime / total_devices if total_devices > 0 else 0
476
-
477
- summary = f"""
478
  Monthly Status for {selected_month}:
479
  - Total Devices: {total_devices}
480
  - Total Usage Hours: {total_usage_hours:.2f}
481
  - Total Downtime Hours: {total_downtime:.2f}
482
- - Average Usage per Device: {avg_usage_per_device:.2f} hours
483
- - Average Downtime per Device: {avg_downtime_per_device:.2f} hours
484
  """
485
- return summary
486
  except Exception as e:
487
  logging.error(f"Failed to generate monthly status: {str(e)}")
488
  return f"Failed to generate monthly status: {str(e)}"
489
 
490
  # Generate PDF content
491
- def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
492
  if not reportlab_available:
493
- logging.error("reportlab not available. PDF generation is disabled.")
494
  return None
495
  try:
496
- logging.info("Starting PDF generation...")
497
  pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
498
  doc = SimpleDocTemplate(pdf_path, pagesize=letter)
499
  styles = getSampleStyleSheet()
500
  story = []
501
 
502
  def safe_paragraph(text, style):
503
- text_str = str(text) if text else ""
504
- return Paragraph(text_str.replace('\n', '<br/>'), style) if text_str else Paragraph("", style)
505
 
506
  story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
507
  story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
508
  story.append(Spacer(1, 12))
509
 
510
- if selected_month != "All" and df is not None:
511
  monthly_status = generate_monthly_status(df, selected_month)
512
  story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
513
  story.append(safe_paragraph(monthly_status, styles['Normal']))
@@ -518,11 +443,29 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
518
  story.append(Spacer(1, 12))
519
 
520
  story.append(Paragraph("Log Preview", styles['Heading2']))
521
- story.append(safe_paragraph(preview, styles['Normal']))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
522
  story.append(Spacer(1, 12))
523
 
524
  story.append(Paragraph("Device Cards", styles['Heading2']))
525
- device_cards_text = device_cards_html.replace('<div>', '').replace('</div>', '\n').replace('<h4>', '').replace('</h4>', '\n').replace('<p>', '').replace('</p>', '\n').replace('<b>', '').replace('</b>', '').replace('<span style="color: green">', '').replace('<span style="color: red">', '').replace('<span style="color: orange">', '').replace('<span style="color: gray">', '').replace('</span>', '') if device_cards_html else "No device cards available."
526
  story.append(safe_paragraph(device_cards_text, styles['Normal']))
527
  story.append(Spacer(1, 12))
528
 
@@ -538,105 +481,31 @@ def generate_pdf_content(summary, preview, anomalies, amc_reminders, insights, d
538
  story.append(safe_paragraph(insights, styles['Normal']))
539
  story.append(Spacer(1, 12))
540
 
541
- story.append(Paragraph("Daily Log Trends Chart", styles['Heading2']))
542
- story.append(Paragraph("[Chart placeholder - see dashboard for Daily Log Trends]" if daily_log_chart is None else "[Chart included in dashboard]", styles['Normal']))
543
- story.append(Spacer(1, 12))
544
-
545
- story.append(Paragraph("Weekly Uptime Percentage Chart", styles['Heading2']))
546
- story.append(Paragraph("[Chart placeholder - see dashboard for Weekly Uptime Percentage]" if weekly_uptime_chart is None else "[Chart included in dashboard]", styles['Normal']))
547
- story.append(Spacer(1, 12))
548
-
549
- story.append(Paragraph("Anomaly Alerts Chart", styles['Heading2']))
550
- story.append(Paragraph("[Chart placeholder - see dashboard for Anomaly Alerts]" if anomaly_alerts_chart is None else "[Chart included in dashboard]", styles['Normal']))
551
- story.append(Spacer(1, 12))
552
-
553
- story.append(Paragraph("Downtime Chart", styles['Heading2']))
554
- story.append(Paragraph("[Chart placeholder - see dashboard for Downtime per Device]" if downtime_chart is None else "[Chart included in dashboard]", styles['Normal']))
555
 
556
  doc.build(story)
557
  logging.info(f"PDF generated at {pdf_path}")
558
  return pdf_path
559
  except Exception as e:
560
- logging.error(f"Failed to generate PDF: {str(e)}", exc_info=True)
561
  return None
562
 
563
- # Send email alert with analysis summary and PDF attachment
564
- def send_email_alert(summary, anomalies, amc_reminders, pdf_path, recipient_email="recipient@example.com"):
565
- try:
566
- # Email configuration
567
- sender_email = "your_email@gmail.com" # Replace with your email
568
- sender_password = "your_app_password" # Replace with your app-specific password
569
- smtp_server = "smtp.gmail.com"
570
- smtp_port = 587
571
-
572
- # Create email message
573
- subject = "LabOps Log Analyzer Report - Analysis Completed"
574
- body = f"""
575
- Dear Recipient,
576
-
577
- The LabOps Log Analyzer has completed its analysis. Below are the key findings:
578
-
579
- **Summary:**
580
- {summary}
581
-
582
- **Anomalies Detected:**
583
- {anomalies}
584
-
585
- **AMC Reminders:**
586
- {amc_reminders}
587
-
588
- The full report is attached as a PDF for your review.
589
-
590
- Regards,
591
- LabOps Team
592
- """
593
-
594
- msg = MIMEMultipart()
595
- msg['From'] = sender_email
596
- msg['To'] = recipient_email
597
- msg['Subject'] = subject
598
- msg.attach(MIMEText(body, 'plain'))
599
-
600
- # Attach the PDF if it exists
601
- if pdf_path and os.path.exists(pdf_path):
602
- with open(pdf_path, 'rb') as f:
603
- pdf_attachment = MIMEApplication(f.read(), _subtype="pdf")
604
- pdf_attachment.add_header(
605
- 'Content-Disposition', 'attachment', filename=os.path.basename(pdf_path)
606
- )
607
- msg.attach(pdf_attachment)
608
- logging.info(f"Attached PDF to email: {pdf_path}")
609
- else:
610
- logging.warning("No PDF file to attach to email.")
611
-
612
- # Send the email
613
- with smtplib.SMTP(smtp_server, smtp_port) as server:
614
- server.starttls()
615
- server.login(sender_email, sender_password)
616
- server.sendmail(sender_email, recipient_email, msg.as_string())
617
-
618
- logging.info(f"Email alert sent to {recipient_email}")
619
- except Exception as e:
620
- logging.error(f"Failed to send email alert: {str(e)}")
621
-
622
- # Main Gradio function
623
  async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
 
624
  try:
625
- start_time = datetime.now()
626
-
627
  if not file_obj:
628
- return "No file uploaded.", "No data to preview.", None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, None, None, None, None, None, None, "Please upload a CSV file to analyze."
629
-
630
  file_path = file_obj.name
631
  current_modified_time = os.path.getmtime(file_path)
632
-
633
  if last_modified_state and current_modified_time == last_modified_state:
634
- return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state, None, None, None, None, None, None, "No changes detected in the file."
635
 
636
- logging.info(f"Processing file: {file_path}, last modified: {current_modified_time}")
637
-
638
  if not file_path.endswith(".csv"):
639
- return "Please upload a CSV file.", "", None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, None, None, None, None, None, None, "Invalid file format. Please upload a CSV file."
640
 
641
  required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
642
  dtypes = {
@@ -648,235 +517,96 @@ async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_ra
648
  "amc_date": "string"
649
  }
650
  df = pd.read_csv(file_path, dtype=dtypes)
651
- # Downsample early if dataset is too large
652
- if len(df) > 5000:
653
- df = df.sample(n=5000, random_state=42)
654
- logging.info(f"Downsampled DataFrame to 5,000 rows immediately after loading.")
655
  missing_columns = [col for col in required_columns if col not in df.columns]
656
  if missing_columns:
657
- return f"Missing columns: {missing_columns}", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, None, None, None, None, None, None, f"Missing required columns: {missing_columns}"
658
-
659
  df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
660
  df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
661
  if df["timestamp"].dt.tz is None:
662
- logging.info("Localizing naive timestamps to IST")
663
  df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
664
  if df.empty:
665
- return "No data available.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, None, None, None, None, None, None, "No data available in the uploaded file."
666
-
667
- logging.info(f"DataFrame before filtering:\n{df.head().to_string()}")
668
-
669
- # Apply filters directly on df
670
- filtered_df = df
671
 
 
 
672
  if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
673
  filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
674
- logging.info(f"After lab_site filter ({lab_site_filter}): {filtered_df.shape[0]} rows")
675
-
676
  if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
677
  filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
678
- logging.info(f"After equipment_type filter ({equipment_type_filter}): {filtered_df.shape[0]} rows")
679
-
680
  if date_range and len(date_range) == 2:
681
  days_start, days_end = date_range
682
  today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
683
  start_date = today + pd.Timedelta(days=days_start)
684
- end_date = today + pd.Timedelta(days=days_end)
685
- end_date = end_date + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
686
- logging.info(f"Applying date range filter: {start_date} to {end_date}")
687
  filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
688
- logging.info(f"After date range filter: {filtered_df.shape[0]} rows")
689
-
690
  if month_filter and month_filter != "All":
691
  selected_date = pd.to_datetime(month_filter, format="%B %Y")
692
  filtered_df = filtered_df[
693
  (filtered_df['timestamp'].dt.year == selected_date.year) &
694
  (filtered_df['timestamp'].dt.month == selected_date.month)
695
  ]
696
- logging.info(f"After month filter ({month_filter}): {filtered_df.shape[0]} rows")
697
 
698
  if filtered_df.empty:
699
- logging.warning("Filtered DataFrame is empty after applying filters.")
700
- return "No data after applying filters.", None, None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, None, None, None, None, None, None, "No data available after applying filters."
701
-
702
- logging.info(f"Filtered DataFrame before AMC check:\n{filtered_df[['device_id', 'amc_date']].to_string()}")
703
 
704
- if len(filtered_df) > 1000:
705
- filtered_df = filtered_df.sample(n=1000, random_state=42)
706
- logging.info(f"Downsampled filtered DataFrame to 1,000 rows for chart generation.")
707
-
708
- # Pre-aggregate data for charts
709
- agg_data = {
710
- 'usage_per_device': filtered_df.groupby("device_id")["usage_hours"].sum().reset_index(),
711
- 'downtime_per_device': filtered_df.groupby("device_id")["downtime"].sum().reset_index(),
712
- }
713
 
714
  # Run tasks concurrently
715
- with ThreadPoolExecutor(max_workers=3) as executor:
716
- future_summary_insights = executor.submit(generate_summary_and_insights, filtered_df)
717
  future_anomalies = executor.submit(detect_anomalies, filtered_df)
718
  future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
719
- future_usage_chart = executor.submit(create_usage_chart, agg_data)
720
- future_downtime_chart = executor.submit(create_downtime_chart, agg_data)
 
721
  future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
722
  future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
 
723
  future_device_cards = executor.submit(generate_device_cards, filtered_df)
 
724
 
725
- summary, insights = future_summary_insights.result()
726
- summary = f"Step 1: Summary Report\n{summary}"
727
- insights = f"Dashboard Insights (AI)\n{insights}"
728
  anomalies, anomalies_df = future_anomalies.result()
729
  anomalies = f"Anomaly Detection\n{anomalies}"
730
  amc_reminders, reminders_df = future_amc.result()
731
  amc_reminders = f"AMC Reminders\n{amc_reminders}"
 
732
  usage_chart = future_usage_chart.result()
733
  downtime_chart = future_downtime_chart.result()
734
  daily_log_chart = future_daily_log_chart.result()
735
  weekly_uptime_chart = future_weekly_uptime_chart.result()
 
736
  device_cards = future_device_cards.result()
737
 
738
- # Generate Anomaly Alerts chart after anomalies are detected
739
- anomaly_alerts_chart = create_anomaly_alerts_chart(filtered_df, anomalies_df)
740
 
741
- # Generate the log preview as an HTML table
742
- preview_html = """
743
- <style>
744
- .log-preview-table {
745
- width: 100%;
746
- border-collapse: collapse;
747
- font-family: Arial, sans-serif;
748
- margin-top: 10px;
749
- }
750
- .log-preview-table th, .log-preview-table td {
751
- border: 1px solid #ddd;
752
- padding: 8px;
753
- text-align: left;
754
- }
755
- .log-preview-table th {
756
- background-color: #4ECDC4;
757
- color: white;
758
- }
759
- .log-preview-table tr:nth-child(even) {
760
- background-color: #f2f2f2;
761
- }
762
- .log-preview-table tr:hover {
763
- background-color: #ddd;
764
- }
765
- </style>
766
- <h3>Step 2: Log Preview (First 5 Rows)</h3>
767
- <table class='log-preview-table'>
768
- <thead>
769
- <tr>
770
- <th>Row</th>
771
- <th>Device ID</th>
772
- <th>Log Type</th>
773
- <th>Status</th>
774
- <th>Timestamp</th>
775
- <th>Usage Hours</th>
776
- <th>Downtime</th>
777
- <th>AMC Date</th>
778
- </tr>
779
- </thead>
780
- <tbody>
781
- """
782
- if filtered_df.empty:
783
- preview_html += "<tr><td colspan='8'>No data to preview.</td></tr>"
784
- else:
785
- for idx, row in filtered_df.head(5).iterrows():
786
- preview_html += f"""
787
- <tr>
788
- <td>{idx + 1}</td>
789
- <td>{row['device_id']}</td>
790
- <td>{row['log_type']}</td>
791
- <td>{row['status']}</td>
792
- <td>{row['timestamp']}</td>
793
- <td>{row['usage_hours']}</td>
794
- <td>{row['downtime']}</td>
795
- <td>{row['amc_date']}</td>
796
- </tr>
797
- """
798
- preview_html += """
799
- </tbody>
800
- </table>
801
- """
802
-
803
- preview_lines = ["Step 2: Log Preview (First 5 Rows)"]
804
- for idx, row in filtered_df.head(5).iterrows():
805
- preview_lines.append(
806
- f"Row {idx + 1}: Device ID: {row['device_id']}, "
807
- f"Log Type: {row['log_type']}, Status: {row['status']}, "
808
- f"Timestamp: {row['timestamp']}, Usage Hours: {row['usage_hours']}, "
809
- f"Downtime: {row['downtime']}, AMC Date: {row['amc_date']}"
810
- )
811
- preview_text = "\n".join(preview_lines)
812
-
813
- # Auto-generate PDF after analysis
814
- pdf_file = None
815
- status_msg = "Analysis completed successfully."
816
- if all([summary, preview_text, anomalies, amc_reminders, insights, device_cards, filtered_df is not None]):
817
- pdf_file = generate_pdf_content(
818
- summary, preview_text, anomalies, amc_reminders, insights, device_cards,
819
- daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart,
820
- filtered_df, month_filter
821
- )
822
- if pdf_file:
823
- status_msg = "Analysis completed successfully. PDF report generated and available for download."
824
- else:
825
- status_msg = "Analysis completed successfully, but failed to generate PDF. Check logs for details."
826
- else:
827
- status_msg = "Analysis completed, but some data is missing for PDF generation."
828
-
829
- # Send email alert
830
- send_email_alert(summary, anomalies, amc_reminders, pdf_file)
831
-
832
- elapsed_time = (datetime.now() - start_time).total_seconds()
833
  logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
834
  if elapsed_time > 10:
835
  logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
836
 
837
- return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time, summary, preview_text, anomalies, amc_reminders, insights, device_cards, filtered_df, status_msg)
838
  except Exception as e:
839
  logging.error(f"Failed to process file: {str(e)}")
840
- return f"Error: {str(e)}", None, None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, None, None, None, None, None, None, f"Failed to process file: {str(e)}"
841
 
842
- # Update filter options
843
  def update_filters(file_obj):
844
  if not file_obj:
845
- logging.info("No file uploaded for filter update, returning default options.")
846
  return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
847
-
848
  try:
849
- logging.info(f"Attempting to read CSV file: {file_obj.name}")
850
  with open(file_obj.name, 'rb') as f:
851
  csv_content = f.read().decode('utf-8')
852
  df = pd.read_csv(io.StringIO(csv_content))
853
  df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
854
- logging.info(f"CSV file read successfully. Columns found: {list(df.columns)}")
855
-
856
- lab_site_options = ['All']
857
- if 'lab_site' in df.columns:
858
- unique_lab_sites = df['lab_site'].dropna().astype(str).unique().tolist()
859
- lab_site_options.extend([site for site in unique_lab_sites if site.strip()])
860
- logging.info(f"Lab site options extracted: {lab_site_options}")
861
- else:
862
- logging.warning("Column 'lab_site' not found in CSV.")
863
-
864
- equipment_type_options = ['All']
865
- if 'equipment_type' in df.columns:
866
- unique_equipment_types = df['equipment_type'].dropna().astype(str).unique().tolist()
867
- equipment_type_options.extend([equip for equip in unique_equipment_types if equip.strip()])
868
- logging.info(f"Equipment type options extracted: {equipment_type_options}")
869
- else:
870
- logging.warning("Column 'equipment_type' not found in CSV.")
871
-
872
- month_options = ['All']
873
- if 'timestamp' in df.columns:
874
- df['month_year'] = df['timestamp'].dt.strftime('%B %Y')
875
- unique_months = df['month_year'].dropna().unique().tolist()
876
- month_options.extend(sorted(unique_months))
877
- logging.info(f"Month options extracted: {month_options}")
878
- else:
879
- logging.warning("Column 'timestamp' not found in CSV.")
880
 
881
  return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
882
  except Exception as e:
@@ -893,74 +623,39 @@ try:
893
  .dashboard-section h3 {font-size: 18px; margin-bottom: 2px;}
894
  .dashboard-section p {margin: 1px 0; line-height: 1.2;}
895
  .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
 
 
 
 
896
  """) as iface:
897
- gr.Markdown("<h1>LabOps Log Analyzer Dashboard</h1>")
898
- gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data. A PDF report will be generated automatically, and an email alert will be sent.")
899
 
900
  last_modified_state = gr.State(value=None)
901
- summary_state = gr.State()
902
- preview_state = gr.State()
903
- anomalies_state = gr.State()
904
- amc_reminders_state = gr.State()
905
- insights_state = gr.State()
906
- device_cards_state = gr.State()
907
- df_state = gr.State()
908
 
909
  with gr.Row():
910
  with gr.Column(scale=1):
911
  file_input = gr.File(label="Upload Logs (CSV)", file_types=[".csv"])
912
-
913
  with gr.Group():
914
  gr.Markdown("### Filters")
915
- lab_site_filter = gr.Dropdown(
916
- label="Lab Site",
917
- choices=['All'],
918
- value='All',
919
- interactive=True
920
- )
921
- equipment_type_filter = gr.Dropdown(
922
- label="Equipment Type",
923
- choices=['All'],
924
- value='All',
925
- interactive=True
926
- )
927
- date_range_filter = gr.Slider(
928
- label="Date Range (Days from Today)",
929
- minimum=-365,
930
- maximum=0,
931
- step=1,
932
- value=[-30, 0],
933
- info="Select the range of days relative to today (e.g., -30 to 0 for the last 30 days)."
934
- )
935
- month_filter = gr.Dropdown(
936
- label="Select Month for Report",
937
- choices=['All'],
938
- value='All',
939
- interactive=True
940
- )
941
-
942
  submit_button = gr.Button("Analyze", variant="primary")
943
 
944
  with gr.Column(scale=2):
945
  with gr.Group(elem_classes="dashboard-container"):
946
  gr.Markdown("<div class='dashboard-title'>Analysis Results</div>")
947
-
948
- with gr.Group(elem_classes="dashboard-section"):
949
- gr.Markdown("### Status Message")
950
- status_message = gr.Markdown("Please upload a CSV file and click 'Analyze' to begin.")
951
-
952
  with gr.Group(elem_classes="dashboard-section"):
953
  gr.Markdown("### Step 1: Summary Report")
954
  summary_output = gr.Markdown()
955
-
956
  with gr.Group(elem_classes="dashboard-section"):
957
  gr.Markdown("### Step 2: Log Preview")
958
  preview_output = gr.HTML()
959
-
960
  with gr.Group(elem_classes="dashboard-section"):
961
  gr.Markdown("### Device Cards")
962
  device_cards_output = gr.HTML()
963
-
964
  with gr.Group(elem_classes="dashboard-section"):
965
  gr.Markdown("### Charts")
966
  with gr.Tab("Usage Hours per Device"):
@@ -973,19 +668,15 @@ try:
973
  weekly_uptime_output = gr.Plot()
974
  with gr.Tab("Anomaly Alerts"):
975
  anomaly_alerts_output = gr.Plot()
976
-
977
  with gr.Group(elem_classes="dashboard-section"):
978
  gr.Markdown("### Step 4: Anomaly Detection")
979
  anomaly_output = gr.Markdown()
980
-
981
  with gr.Group(elem_classes="dashboard-section"):
982
  gr.Markdown("### Step 5: AMC Reminders")
983
  amc_output = gr.Markdown()
984
-
985
  with gr.Group(elem_classes="dashboard-section"):
986
  gr.Markdown("### Step 6: Insights (AI)")
987
  insights_output = gr.Markdown()
988
-
989
  with gr.Group(elem_classes="dashboard-section"):
990
  gr.Markdown("### Export Report")
991
  pdf_output = gr.File(label="Download Monthly Status Report as PDF")
@@ -999,37 +690,8 @@ try:
999
 
1000
  submit_button.click(
1001
  fn=process_logs,
1002
- inputs=[
1003
- file_input,
1004
- lab_site_filter,
1005
- equipment_type_filter,
1006
- date_range_filter,
1007
- month_filter,
1008
- last_modified_state
1009
- ],
1010
- outputs=[
1011
- summary_output,
1012
- preview_output,
1013
- usage_chart_output,
1014
- device_cards_output,
1015
- daily_log_trends_output,
1016
- weekly_uptime_output,
1017
- anomaly_alerts_output,
1018
- downtime_chart_output,
1019
- anomaly_output,
1020
- amc_output,
1021
- insights_output,
1022
- pdf_output,
1023
- last_modified_state,
1024
- summary_state,
1025
- preview_state,
1026
- anomalies_state,
1027
- amc_reminders_state,
1028
- insights_state,
1029
- device_cards_state,
1030
- df_state,
1031
- status_message
1032
- ]
1033
  )
1034
 
1035
  logging.info("Gradio interface initialized successfully")
 
 
 
 
1
  import gradio as gr
2
  import pandas as pd
3
  from datetime import datetime, timedelta
4
  import logging
5
  import plotly.express as px
6
  from sklearn.ensemble import IsolationForest
7
+ from transformers import pipeline
8
+ import torch
9
  from concurrent.futures import ThreadPoolExecutor
10
+ from simple_salesforce import Salesforce
11
  import os
12
+ import json
13
  import io
14
+ import time
 
 
 
15
 
16
  # Configure logging
17
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
18
 
19
+ # Salesforce configuration
20
+ try:
21
+ sf = Salesforce(
22
+ username='multi-devicelabopsdashboard@sathkrutha.com',
23
+ password='Team@1234',
24
+ security_token=os.getenv('SF_SECURITY_TOKEN', ''),
25
+ domain='login'
26
+ )
27
+ logging.info("Salesforce connection established")
28
+ except Exception as e:
29
+ logging.error(f"Failed to connect to Salesforce: {str(e)}")
30
+ sf = None
31
+
32
  # Try to import reportlab
33
  try:
34
  from reportlab.lib.pagesizes import letter
35
+ from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle
36
  from reportlab.lib.styles import getSampleStyleSheet
37
+ from reportlab.lib import colors
38
  reportlab_available = True
39
  logging.info("reportlab module successfully imported")
40
  except ImportError:
41
  logging.warning("reportlab module not found. PDF generation disabled.")
42
  reportlab_available = False
43
 
44
+ # Preload Hugging Face model with optimization
45
+ logging.info("Preloading Hugging Face model...")
46
+ try:
47
+ device = 0 if torch.cuda.is_available() else -1
48
+ # Use a smaller model for faster inference
49
+ summarizer = pipeline(
50
+ "summarization",
51
+ model="t5-small",
52
+ device=device,
53
+ max_length=50,
54
+ min_length=10,
55
+ num_beams=2
56
+ )
57
+ logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
58
+ except Exception as e:
59
+ logging.error(f"Failed to preload model: {str(e)}")
60
+ raise e
61
+
62
+ # Cache picklist values at startup
63
+ def get_picklist_values(field_name):
64
+ if sf is None:
65
+ return []
66
+ try:
67
+ obj_desc = sf.SmartLog__c.describe()
68
+ for field in obj_desc['fields']:
69
+ if field['name'] == field_name:
70
+ return [value['value'] for value in field['picklistValues'] if value['active']]
71
+ return []
72
+ except Exception as e:
73
+ logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
74
+ return []
75
+
76
+ status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
77
+ log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
78
+ logging.info(f"Valid Status__c values: {status_values}")
79
+ logging.info(f"Valid Log_Type__c values: {log_type_values}")
80
+
81
+ # Map invalid picklist values
82
+ picklist_mapping = {
83
+ 'Status__c': {
84
+ 'normal': 'Active',
85
+ 'error': 'Inactive',
86
+ 'warning': 'Pending',
87
+ 'ok': 'Active',
88
+ 'failed': 'Inactive'
89
+ },
90
+ 'Log_Type__c': {
91
+ 'maint': 'Smart Log',
92
+ 'error': 'Cell Analysis',
93
+ 'ops': 'UV Verification',
94
+ 'maintenance': 'Smart Log',
95
+ 'cell': 'Cell Analysis',
96
+ 'uv': 'UV Verification',
97
+ 'weight log': 'Smart Log'
98
+ }
99
+ }
100
+
101
+ # Cache folder ID
102
+ def get_folder_id(folder_name):
103
+ if sf is None:
104
+ return None
105
+ try:
106
+ query = f"SELECT Id FROM Folder WHERE Name = '{folder_name}' AND Type = 'Report'"
107
+ result = sf.query(query)
108
+ if result['totalSize'] > 0:
109
+ folder_id = result['records'][0]['Id']
110
+ logging.info(f"Found folder ID for '{folder_name}': {folder_id}")
111
+ return folder_id
112
+ else:
113
+ logging.error(f"Folder '{folder_name}' not found in Salesforce.")
114
+ return None
115
+ except Exception as e:
116
+ logging.error(f"Failed to fetch folder ID for '{folder_name}': {str(e)}")
117
+ return None
118
+
119
+ LABOPS_REPORTS_FOLDER_ID = get_folder_id('LabOps Reports')
120
+
121
+ # Salesforce report creation
122
+ def create_salesforce_reports(df):
123
+ if sf is None or not LABOPS_REPORTS_FOLDER_ID:
124
+ return
125
+ try:
126
+ timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
127
+ reports = [
128
+ {
129
+ "reportMetadata": {
130
+ "name": f"SmartLog_Usage_Report_{timestamp}",
131
+ "developerName": f"SmartLog_Usage_Report_{timestamp}",
132
+ "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
133
+ "reportFormat": "TABULAR",
134
+ "reportBooleanFilter": None,
135
+ "reportFilters": [],
136
+ "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.Usage_Hours__c"],
137
+ "folderId": LABOPS_REPORTS_FOLDER_ID
138
+ }
139
+ },
140
+ {
141
+ "reportMetadata": {
142
+ "name": f"SmartLog_AMC_Reminders_{timestamp}",
143
+ "developerName": f"SmartLog_AMC_Reminders_{timestamp}",
144
+ "reportType": {"type": "CustomEntity", "value": "SmartLog__c"},
145
+ "reportFormat": "TABULAR",
146
+ "reportBooleanFilter": None,
147
+ "reportFilters": [],
148
+ "detailColumns": ["SmartLog__c.Device_Id__c", "SmartLog__c.AMC_Date__c"],
149
+ "folderId": LABOPS_REPORTS_FOLDER_ID
150
+ }
151
+ }
152
+ ]
153
+ for report in reports:
154
+ sf.restful('analytics/reports', method='POST', json=report)
155
+ logging.info("Salesforce reports created")
156
+ except Exception as e:
157
+ logging.error(f"Failed to create Salesforce reports: {str(e)}")
158
+
159
+ # Save to Salesforce
160
+ def save_to_salesforce(df, reminders_df):
161
+ if sf is None:
162
+ return
163
+ try:
164
+ current_date = datetime.now()
165
+ next_30_days = current_date + timedelta(days=30)
166
+ records = []
167
+ reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
168
+
169
+ for _, row in df.iterrows():
170
+ status = str(row['status'])
171
+ log_type = str(row['log_type'])
172
+ status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
173
+ log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
174
+ if status is None or log_type is None:
175
+ continue
176
+
177
+ amc_date_str = None
178
+ if pd.notna(row['amc_date']):
179
+ try:
180
+ amc_date = pd.to_datetime(row['amc_date']).strftime('%Y-%m-%d')
181
+ amc_date_dt = datetime.strptime(amc_date, '%Y-%m-%d')
182
+ if status == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
183
+ logging.info(f"AMC Reminder for Device ID {row['device_id']}")
184
+ except:
185
+ amc_date_str = None
186
+
187
+ record = {
188
+ 'Device_Id__c': str(row['device_id'])[:50],
189
+ 'Log_Type__c': log_type,
190
+ 'Status__c': status,
191
+ 'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
192
+ 'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
193
+ 'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
194
+ 'AMC_Date__c': amc_date_str
195
+ }
196
+ if row['device_id'] not in reminder_device_ids:
197
+ records.append(record)
198
+
199
+ if records:
200
+ sf.bulk.SmartLog__c.insert(records)
201
+ logging.info(f"Saved {len(records)} records to Salesforce")
202
+ except Exception as e:
203
+ logging.error(f"Failed to save to Salesforce: {str(e)}")
204
+
205
+ # Summarize logs
206
+ def summarize_logs(df):
207
  try:
208
  total_devices = df["device_id"].nunique()
209
  most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
210
+ prompt = f"Maintenance logs: {total_devices} devices. Most used: {most_used}."
211
+ summary = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
212
+ return summary
 
213
  except Exception as e:
214
+ logging.error(f"Summary generation failed: {str(e)}")
215
+ return f"Failed to generate summary: {str(e)}"
216
 
217
  # Anomaly detection
218
  def detect_anomalies(df):
219
  try:
220
  if "usage_hours" not in df.columns or "downtime" not in df.columns:
221
  return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
 
 
222
  features = df[["usage_hours", "downtime"]].fillna(0)
223
+ if len(features) > 500: # Reduced sample size
224
+ features = features.sample(n=500, random_state=42)
225
+ iso_forest = IsolationForest(contamination=0.1, random_state=42)
226
  df["anomaly"] = iso_forest.fit_predict(features)
227
  anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
228
  if anomalies.empty:
229
  return "No anomalies detected.", anomalies
230
+ return "\n".join([f"- Device ID: {row['device_id']}, Usage: {row['usage_hours']}, Downtime: {row['downtime']}, Timestamp: {row['timestamp']}" for _, row in anomalies.head(5).iterrows()]), anomalies
 
 
 
 
 
 
231
  except Exception as e:
232
  logging.error(f"Anomaly detection failed: {str(e)}")
233
  return f"Anomaly detection failed: {str(e)}", pd.DataFrame()
 
235
  # AMC reminders
236
  def check_amc_reminders(df, current_date):
237
  try:
 
238
  if "device_id" not in df.columns or "amc_date" not in df.columns:
 
239
  return "AMC reminders require 'device_id' and 'amc_date' columns.", pd.DataFrame()
 
240
  df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
241
+ current_date = pd.to_datetime(current_date)
 
 
 
 
 
 
242
  df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
 
 
243
  reminders = df[(df["days_to_amc"] >= 0) & (df["days_to_amc"] <= 30)][["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]]
244
  if reminders.empty:
 
245
  return "No AMC reminders due within the next 30 days.", reminders
246
+ return "\n".join([f"- Device ID: {row['device_id']}, AMC Date: {row['amc_date']}" for _, row in reminders.head(5).iterrows()]), reminders
 
 
 
 
 
247
  except Exception as e:
248
  logging.error(f"AMC reminder generation failed: {str(e)}")
249
  return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
250
 
251
+ # Dashboard insights
252
+ def generate_dashboard_insights(df):
253
+ try:
254
+ total_devices = df["device_id"].nunique()
255
+ avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
256
+ prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
257
+ insights = summarizer(prompt, max_length=50, min_length=10, do_sample=False)[0]["summary_text"]
258
+ return insights
259
+ except Exception as e:
260
+ logging.error(f"Dashboard insights generation failed: {str(e)}")
261
+ return f"Dashboard insights generation failed: {str(e)}"
262
+
263
  # Create usage chart
264
+ def create_usage_chart(df):
265
  try:
266
+ if df.empty:
 
 
 
267
  return None
268
+ usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
269
  if len(usage_data) > 5:
270
  usage_data = usage_data.nlargest(5, "usage_hours")
 
 
 
 
 
 
 
 
271
  fig = px.bar(
272
  usage_data,
273
  x="device_id",
274
  y="usage_hours",
275
+ title="Usage Hours per Device",
276
+ labels={"device_id": "Device ID", "usage_hours": "Usage Hours"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
277
  )
278
+ fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
279
  return fig
280
  except Exception as e:
281
  logging.error(f"Failed to create usage chart: {str(e)}")
282
  return None
283
 
284
+ # Create downtime chart
285
+ def create_downtime_chart(df):
286
  try:
287
+ downtime_data = df.groupby("device_id")["downtime"].sum().reset_index()
 
 
 
 
288
  if len(downtime_data) > 5:
289
  downtime_data = downtime_data.nlargest(5, "downtime")
 
 
 
 
 
 
 
 
290
  fig = px.bar(
291
  downtime_data,
292
  x="device_id",
293
  y="downtime",
294
+ title="Downtime per Device",
295
+ labels={"device_id": "Device ID", "downtime": "Downtime (Hours)"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
296
  )
297
+ fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
298
  return fig
299
  except Exception as e:
300
  logging.error(f"Failed to create downtime chart: {str(e)}")
301
  return None
302
 
303
+ # Create daily log trends chart
304
  def create_daily_log_trends_chart(df):
305
  try:
 
 
 
 
 
306
  df['date'] = df['timestamp'].dt.date
307
+ daily_logs = df.groupby('date').size().reset_index(name='log_count')
308
+ fig = px.line(
309
+ daily_logs,
 
310
  x='date',
311
  y='log_count',
312
  title="Daily Log Trends",
313
  labels={"date": "Date", "log_count": "Number of Logs"}
314
  )
315
+ fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
316
  return fig
317
  except Exception as e:
318
+ logging.error(f"Failed to create daily log trends chart: {str(e)}")
319
  return None
320
 
321
+ # Create weekly uptime chart
322
  def create_weekly_uptime_chart(df):
323
  try:
324
+ df['week'] = df['timestamp'].dt.isocalendar().week
 
 
 
 
 
 
 
 
 
 
 
325
  df['year'] = df['timestamp'].dt.year
326
  weekly_data = df.groupby(['year', 'week']).agg({
327
+ 'usage_hours': 'sum',
328
  'downtime': 'sum'
329
  }).reset_index()
330
+ weekly_data['uptime_percent'] = (weekly_data['usage_hours'] / (weekly_data['usage_hours'] + weekly_data['downtime'])) * 100
331
+ weekly_data['year_week'] = weekly_data['year'].astype(str) + '-W' + weekly_data['week'].astype(str)
 
 
 
 
 
 
 
 
 
 
 
332
  fig = px.bar(
333
  weekly_data,
334
+ x='year_week',
335
+ y='uptime_percent',
336
  title="Weekly Uptime Percentage",
337
+ labels={"year_week": "Year-Week", "uptime_percent": "Uptime %"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  )
339
+ fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
340
  return fig
341
  except Exception as e:
342
+ logging.error(f"Failed to create weekly uptime chart: {str(e)}")
343
  return None
344
 
345
+ # Create anomaly alerts chart
346
+ def create_anomaly_alerts_chart(anomalies_df):
347
  try:
348
+ if anomalies_df.empty:
 
349
  return None
350
+ anomalies_df['date'] = anomalies_df['timestamp'].dt.date
351
+ anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
 
 
 
352
  fig = px.scatter(
353
+ anomaly_counts,
354
+ x='date',
355
+ y='anomaly_count',
356
+ title="Anomaly Alerts Over Time",
357
+ labels={"date": "Date", "anomaly_count": "Number of Anomalies"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
358
  )
359
+ fig.update_layout(title_font_size=16, margin=dict(l=20, r=20, t=40, b=20))
360
  return fig
361
  except Exception as e:
362
+ logging.error(f"Failed to create anomaly alerts chart: {str(e)}")
363
  return None
364
 
365
+ # Generate device cards
366
  def generate_device_cards(df):
367
  try:
368
  if df.empty:
 
369
  return '<p>No devices available to display.</p>'
370
+ device_stats = df.groupby('device_id').agg({
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
371
  'status': 'last',
372
  'timestamp': 'max',
373
  }).reset_index()
374
+ device_stats['count'] = df.groupby('device_id').size().reindex(device_stats['device_id']).values
 
 
 
 
 
 
 
375
  device_stats['health'] = device_stats['status'].map({
376
  'Active': 'Healthy',
377
  'Inactive': 'Unhealthy',
378
  'Pending': 'Warning'
379
  }).fillna('Unknown')
 
380
  cards_html = '<div style="display: flex; flex-wrap: wrap; gap: 20px;">'
381
  for _, row in device_stats.iterrows():
382
+ health_color = {'Healthy': 'green', 'Unhealthy': 'red', 'Warning': 'orange', 'Unknown': 'gray'}.get(row['health'], 'gray')
 
 
 
 
 
383
  timestamp_str = str(row['timestamp']) if pd.notna(row['timestamp']) else 'Unknown'
384
+ cards_html += f"""
385
  <div style="border: 1px solid #e0e0e0; padding: 10px; border-radius: 5px; width: 200px;">
386
  <h4>Device: {row['device_id']}</h4>
387
  <p><b>Health:</b> <span style="color: {health_color}">{row['health']}</span></p>
 
389
  <p><b>Last Log:</b> {timestamp_str}</p>
390
  </div>
391
  """
 
392
  cards_html += '</div>'
 
393
  return cards_html
394
  except Exception as e:
395
+ logging.error(f"Failed to generate device cards: {str(e)}")
396
  return f'<p>Error generating device cards: {str(e)}</p>'
397
 
398
+ # Generate monthly status
399
  def generate_monthly_status(df, selected_month):
400
  try:
401
  total_devices = df['device_id'].nunique()
402
  total_usage_hours = df['usage_hours'].sum()
403
  total_downtime = df['downtime'].sum()
404
+ avg_usage = total_usage_hours / total_devices if total_devices > 0 else 0
405
+ avg_downtime = total_downtime / total_devices if total_devices > 0 else 0
406
+ return f"""
 
407
  Monthly Status for {selected_month}:
408
  - Total Devices: {total_devices}
409
  - Total Usage Hours: {total_usage_hours:.2f}
410
  - Total Downtime Hours: {total_downtime:.2f}
411
+ - Average Usage per Device: {avg_usage:.2f} hours
412
+ - Average Downtime per Device: {avg_downtime:.2f} hours
413
  """
 
414
  except Exception as e:
415
  logging.error(f"Failed to generate monthly status: {str(e)}")
416
  return f"Failed to generate monthly status: {str(e)}"
417
 
418
  # Generate PDF content
419
+ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards_html, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, df, selected_month):
420
  if not reportlab_available:
 
421
  return None
422
  try:
 
423
  pdf_path = f"monthly_status_report_{selected_month.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
424
  doc = SimpleDocTemplate(pdf_path, pagesize=letter)
425
  styles = getSampleStyleSheet()
426
  story = []
427
 
428
  def safe_paragraph(text, style):
429
+ return Paragraph(str(text).replace('\n', '<br/>'), style) if text else Paragraph("", style)
 
430
 
431
  story.append(Paragraph("LabOps Monthly Status Report", styles['Title']))
432
  story.append(Paragraph(f"Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", styles['Normal']))
433
  story.append(Spacer(1, 12))
434
 
435
+ if selected_month != "All":
436
  monthly_status = generate_monthly_status(df, selected_month)
437
  story.append(Paragraph("Monthly Status Summary", styles['Heading2']))
438
  story.append(safe_paragraph(monthly_status, styles['Normal']))
 
443
  story.append(Spacer(1, 12))
444
 
445
  story.append(Paragraph("Log Preview", styles['Heading2']))
446
+ if not preview_df.empty:
447
+ data = [preview_df.columns.tolist()] + preview_df.head(5).values.tolist()
448
+ table = Table(data)
449
+ table.setStyle(TableStyle([
450
+ ('BACKGROUND', (0, 0), (-1, 0), colors.grey),
451
+ ('TEXTCOLOR', (0, 0), (-1, 0), colors.whitesmoke),
452
+ ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
453
+ ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
454
+ ('FONTSIZE', (0, 0), (-1, 0), 12),
455
+ ('BOTTOMPADDING', (0, 0), (-1, 0), 12),
456
+ ('BACKGROUND', (0, 1), (-1, -1), colors.beige),
457
+ ('TEXTCOLOR', (0, 1), (-1, -1), colors.black),
458
+ ('FONTNAME', (0, 1), (-1, -1), 'Helvetica'),
459
+ ('FONTSIZE', (0, 1), (-1, -1), 10),
460
+ ('GRID', (0, 0), (-1, -1), 1, colors.black)
461
+ ]))
462
+ story.append(table)
463
+ else:
464
+ story.append(safe_paragraph("No preview available.", styles['Normal']))
465
  story.append(Spacer(1, 12))
466
 
467
  story.append(Paragraph("Device Cards", styles['Heading2']))
468
+ device_cards_text = device_cards_html.replace('<div>', '').replace('</div>', '\n').replace('<h4>', '').replace('</h4>', '\n').replace('<p>', '').replace('</p>', '\n').replace('<b>', '').replace('</b>', '').replace('<span style="color: green">', '').replace('<span style="color: red">', '').replace('<span style="color: orange">', '').replace('<span style="color: gray">', '').replace('</span>', '')
469
  story.append(safe_paragraph(device_cards_text, styles['Normal']))
470
  story.append(Spacer(1, 12))
471
 
 
481
  story.append(safe_paragraph(insights, styles['Normal']))
482
  story.append(Spacer(1, 12))
483
 
484
+ story.append(Paragraph("Charts", styles['Heading2']))
485
+ story.append(Paragraph("[Chart placeholders - see dashboard for visuals]", styles['Normal']))
 
 
 
 
 
 
 
 
 
 
 
 
486
 
487
  doc.build(story)
488
  logging.info(f"PDF generated at {pdf_path}")
489
  return pdf_path
490
  except Exception as e:
491
+ logging.error(f"Failed to generate PDF: {str(e)}")
492
  return None
493
 
494
+ # Main processing function
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
495
  async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, month_filter, last_modified_state):
496
+ start_time = time.time()
497
  try:
 
 
498
  if not file_obj:
499
+ return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state
500
+
501
  file_path = file_obj.name
502
  current_modified_time = os.path.getmtime(file_path)
 
503
  if last_modified_state and current_modified_time == last_modified_state:
504
+ return None, None, None, None, None, None, None, None, None, None, None, None, last_modified_state
505
 
506
+ logging.info(f"Processing file: {file_path}")
 
507
  if not file_path.endswith(".csv"):
508
+ return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state
509
 
510
  required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
511
  dtypes = {
 
517
  "amc_date": "string"
518
  }
519
  df = pd.read_csv(file_path, dtype=dtypes)
 
 
 
 
520
  missing_columns = [col for col in required_columns if col not in df.columns]
521
  if missing_columns:
522
+ return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
523
+
524
  df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
525
  df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
526
  if df["timestamp"].dt.tz is None:
 
527
  df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
528
  if df.empty:
529
+ return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
 
 
 
 
530
 
531
+ # Apply filters
532
+ filtered_df = df.copy()
533
  if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
534
  filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
 
 
535
  if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
536
  filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
 
 
537
  if date_range and len(date_range) == 2:
538
  days_start, days_end = date_range
539
  today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
540
  start_date = today + pd.Timedelta(days=days_start)
541
+ end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
 
 
542
  filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
 
 
543
  if month_filter and month_filter != "All":
544
  selected_date = pd.to_datetime(month_filter, format="%B %Y")
545
  filtered_df = filtered_df[
546
  (filtered_df['timestamp'].dt.year == selected_date.year) &
547
  (filtered_df['timestamp'].dt.month == selected_date.month)
548
  ]
 
549
 
550
  if filtered_df.empty:
551
+ return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state
 
 
 
552
 
553
+ # Generate table for preview
554
+ preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
555
+ preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
 
 
 
 
 
556
 
557
  # Run tasks concurrently
558
+ with ThreadPoolExecutor(max_workers=6) as executor:
559
+ future_summary = executor.submit(summarize_logs, filtered_df)
560
  future_anomalies = executor.submit(detect_anomalies, filtered_df)
561
  future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
562
+ future_insights = executor.submit(generate_dashboard_insights, filtered_df)
563
+ future_usage_chart = executor.submit(create_usage_chart, filtered_df)
564
+ future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
565
  future_daily_log_chart = executor.submit(create_daily_log_trends_chart, filtered_df)
566
  future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
567
+ future_anomaly_alerts_chart = executor.submit(create_anomaly_alerts_chart, pd.DataFrame())
568
  future_device_cards = executor.submit(generate_device_cards, filtered_df)
569
+ future_reports = executor.submit(create_salesforce_reports, filtered_df)
570
 
571
+ summary = f"Step 1: Summary Report\n{future_summary.result()}"
 
 
572
  anomalies, anomalies_df = future_anomalies.result()
573
  anomalies = f"Anomaly Detection\n{anomalies}"
574
  amc_reminders, reminders_df = future_amc.result()
575
  amc_reminders = f"AMC Reminders\n{amc_reminders}"
576
+ insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
577
  usage_chart = future_usage_chart.result()
578
  downtime_chart = future_downtime_chart.result()
579
  daily_log_chart = future_daily_log_chart.result()
580
  weekly_uptime_chart = future_weekly_uptime_chart.result()
581
+ anomaly_alerts_chart = future_anomaly_alerts_chart.result()
582
  device_cards = future_device_cards.result()
583
 
584
+ save_to_salesforce(filtered_df, reminders_df)
585
+ pdf_file = generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, filtered_df, month_filter)
586
 
587
+ elapsed_time = time.time() - start_time
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
588
  logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
589
  if elapsed_time > 10:
590
  logging.warning(f"Processing time exceeded 10 seconds: {elapsed_time:.2f} seconds")
591
 
592
+ return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, pdf_file, current_modified_time)
593
  except Exception as e:
594
  logging.error(f"Failed to process file: {str(e)}")
595
+ return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state
596
 
597
+ # Update filters
598
  def update_filters(file_obj):
599
  if not file_obj:
 
600
  return gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All'), gr.update(choices=['All'], value='All')
 
601
  try:
 
602
  with open(file_obj.name, 'rb') as f:
603
  csv_content = f.read().decode('utf-8')
604
  df = pd.read_csv(io.StringIO(csv_content))
605
  df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')
606
+
607
+ lab_site_options = ['All'] + [site for site in df['lab_site'].dropna().astype(str).unique().tolist() if site.strip()] if 'lab_site' in df.columns else ['All']
608
+ equipment_type_options = ['All'] + [equip for equip in df['equipment_type'].dropna().astype(str).unique().tolist() if equip.strip()] if 'equipment_type' in df.columns else ['All']
609
+ month_options = ['All'] + sorted(df['timestamp'].dt.strftime('%B %Y').dropna().unique().tolist()) if 'timestamp' in df.columns else ['All']
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
610
 
611
  return gr.update(choices=lab_site_options, value='All'), gr.update(choices=equipment_type_options, value='All'), gr.update(choices=month_options, value='All')
612
  except Exception as e:
 
623
  .dashboard-section h3 {font-size: 18px; margin-bottom: 2px;}
624
  .dashboard-section p {margin: 1px 0; line-height: 1.2;}
625
  .dashboard-section ul {margin: 2px 0; padding-left: 20px;}
626
+ .table {width: 100%; border-collapse: collapse;}
627
+ .table th, .table td {border: 1px solid #ddd; padding: 8px; text-align: left;}
628
+ .table th {background-color: #f2f2f2;}
629
+ .table tr:nth-child(even) {background-color: #f9f9f9;}
630
  """) as iface:
631
+ gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
632
+ gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard with the latest data.")
633
 
634
  last_modified_state = gr.State(value=None)
 
 
 
 
 
 
 
635
 
636
  with gr.Row():
637
  with gr.Column(scale=1):
638
  file_input = gr.File(label="Upload Logs (CSV)", file_types=[".csv"])
 
639
  with gr.Group():
640
  gr.Markdown("### Filters")
641
+ lab_site_filter = gr.Dropdown(label="Lab Site", choices=['All'], value='All', interactive=True)
642
+ equipment_type_filter = gr.Dropdown(label="Equipment Type", choices=['All'], value='All', interactive=True)
643
+ date_range_filter = gr.Slider(label="Date Range (Days from Today)", minimum=-365, maximum=0, step=1, value=[-30, 0])
644
+ month_filter = gr.Dropdown(label="Select Month for Report", choices=['All'], value='All', interactive=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
645
  submit_button = gr.Button("Analyze", variant="primary")
646
 
647
  with gr.Column(scale=2):
648
  with gr.Group(elem_classes="dashboard-container"):
649
  gr.Markdown("<div class='dashboard-title'>Analysis Results</div>")
 
 
 
 
 
650
  with gr.Group(elem_classes="dashboard-section"):
651
  gr.Markdown("### Step 1: Summary Report")
652
  summary_output = gr.Markdown()
 
653
  with gr.Group(elem_classes="dashboard-section"):
654
  gr.Markdown("### Step 2: Log Preview")
655
  preview_output = gr.HTML()
 
656
  with gr.Group(elem_classes="dashboard-section"):
657
  gr.Markdown("### Device Cards")
658
  device_cards_output = gr.HTML()
 
659
  with gr.Group(elem_classes="dashboard-section"):
660
  gr.Markdown("### Charts")
661
  with gr.Tab("Usage Hours per Device"):
 
668
  weekly_uptime_output = gr.Plot()
669
  with gr.Tab("Anomaly Alerts"):
670
  anomaly_alerts_output = gr.Plot()
 
671
  with gr.Group(elem_classes="dashboard-section"):
672
  gr.Markdown("### Step 4: Anomaly Detection")
673
  anomaly_output = gr.Markdown()
 
674
  with gr.Group(elem_classes="dashboard-section"):
675
  gr.Markdown("### Step 5: AMC Reminders")
676
  amc_output = gr.Markdown()
 
677
  with gr.Group(elem_classes="dashboard-section"):
678
  gr.Markdown("### Step 6: Insights (AI)")
679
  insights_output = gr.Markdown()
 
680
  with gr.Group(elem_classes="dashboard-section"):
681
  gr.Markdown("### Export Report")
682
  pdf_output = gr.File(label="Download Monthly Status Report as PDF")
 
690
 
691
  submit_button.click(
692
  fn=process_logs,
693
+ inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, month_filter, last_modified_state],
694
+ outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_chart, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
695
  )
696
 
697
  logging.info("Gradio interface initialized successfully")