# enhanced_anomaly_intelligence_v2.py # TAQATHON 2025 - Production Anomaly Intelligence with Equipment Intelligence # Enhanced for single and batch processing with safety override rules import pandas as pd import numpy as np import joblib import json import re from datetime import datetime from sklearn.metrics.pairwise import cosine_similarity import warnings from typing import Union, List, Dict, Any import time warnings.filterwarnings('ignore') class EnhancedAnomalyIntelligence: """ Enhanced Production-ready Anomaly Intelligence System v2.0 Features: Equipment Intelligence + Safety Override Rules + Conservative Prediction """ def __init__(self): self.models = {} self.model_metadata = None self.safety_rules = None self.embeddings = None self.embedding_metadata = None self.sentence_model = None self._models_loaded = False # Equipment intelligence configuration self.equipment_type_scores = {} self.section_risk_multipliers = {} def _load_models(self): """Load all enhanced models and metadata (called once)""" if self._models_loaded: return print("Loading enhanced models and metadata...") try: # Load enhanced model metadata self.model_metadata = joblib.load('enhanced_model_metadata_v2.joblib') target_columns = self.model_metadata['target_columns'] # Load enhanced trained models for target in target_columns: model_filename = f"enhanced_model_{target.replace(' ', '_').replace('é', 'e')}_v2.joblib" self.models[target] = joblib.load(model_filename) print(f"✓ Loaded {target} model") # Load safety override rules try: with open('safety_override_rules_v2.json', 'r') as f: self.safety_rules = json.load(f) print("✓ Loaded safety override rules") except FileNotFoundError: print("⚠️ Warning: safety_override_rules_v2.json not found - safety rules disabled") self.safety_rules = {} # Load embeddings and metadata for similarity search try: self.embeddings = np.load('anomaly_embeddings.npy') self.embedding_metadata = joblib.load('embedding_metadata.joblib') print("✓ Loaded similarity search embeddings") except FileNotFoundError: print("⚠️ Warning: Embedding files not found - similarity search disabled") self.embeddings = None self.embedding_metadata = None # Load sentence transformer try: from sentence_transformers import SentenceTransformer try: self.sentence_model = SentenceTransformer('dangvantuan/sentence-camembert-large') print("✓ Loaded French CamemBERT model") except: try: self.sentence_model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2') print("✓ Loaded multilingual model") except: self.sentence_model = SentenceTransformer('distiluse-base-multilingual-cased') print("✓ Loaded basic multilingual model") except Exception as e: print(f"⚠️ Warning: Could not load sentence transformer: {e}") self.sentence_model = None # Extract equipment intelligence configuration if 'training_config' in self.model_metadata: training_config = self.model_metadata['training_config'] print("✓ Loaded training configuration") self._models_loaded = True print("✓ All enhanced models loaded successfully") except Exception as e: raise Exception(f"Failed to load enhanced models: {str(e)}") def predict_single(self, anomaly_data: Dict, confidence_threshold: float = 0.7, include_similar: bool = True, format_type: str = 'rich', apply_safety_rules: bool = True) -> Dict: """ Enhanced single anomaly prediction with equipment intelligence and safety rules Args: anomaly_data: Dictionary with anomaly information confidence_threshold: Threshold for flagging manual review include_similar: Whether to include similar anomalies format_type: 'rich' for UI, 'simple' for database apply_safety_rules: Whether to apply safety override rules """ self._load_models() try: # Extract and prepare enhanced features enhanced_features = self._extract_enhanced_features_single(anomaly_data) # Make base predictions predictions, confidences, probabilities = self._predict_criticality(enhanced_features) # Apply safety override rules if enabled if apply_safety_rules and self.safety_rules: predictions = self._apply_safety_override_rules(enhanced_features, predictions) # Calculate enhanced metrics total_criticality = sum(predictions.values()) overall_confidence = np.mean(list(confidences.values())) # Enhanced business logic for manual review needs_review = self._determine_manual_review_need( enhanced_features, predictions, overall_confidence, confidence_threshold ) # Equipment-specific risk assessment equipment_risk_assessment = self._assess_equipment_risk(enhanced_features, predictions) # Find similar anomalies similar_anomalies = [] if include_similar and self.sentence_model is not None: similar_anomalies = self._find_similar_anomalies( anomaly_data.get('Description', ''), top_k=3 ) # Format response based on type if format_type == 'simple': return self._format_simple_response( anomaly_data, predictions, total_criticality, overall_confidence, needs_review, equipment_risk_assessment ) else: return self._format_rich_response( anomaly_data, predictions, confidences, total_criticality, overall_confidence, similar_anomalies, needs_review, confidence_threshold, equipment_risk_assessment, enhanced_features ) except Exception as e: return { 'error': f'Enhanced prediction failed: {str(e)}', 'timestamp': datetime.now().isoformat(), 'input_description': anomaly_data.get('Description', 'N/A') } def predict_batch(self, anomaly_list: List[Dict], confidence_threshold: float = 0.7, include_similar: bool = False, format_type: str = 'simple', apply_safety_rules: bool = True) -> List[Dict]: """ Enhanced batch prediction with equipment intelligence Args: anomaly_list: List of anomaly dictionaries confidence_threshold: Threshold for flagging manual review include_similar: Whether to include similar anomalies (slower for batch) format_type: 'rich' for UI, 'simple' for database apply_safety_rules: Whether to apply safety override rules """ self._load_models() print(f"Processing enhanced batch of {len(anomaly_list)} anomalies...") start_time = time.time() results = [] try: # Extract enhanced features for all anomalies all_features = [] for anomaly_data in anomaly_list: enhanced_features = self._extract_enhanced_features_single(anomaly_data) all_features.append(enhanced_features) # Create batch DataFrame with all enhanced features batch_df = pd.DataFrame(all_features) # Make batch predictions batch_predictions = {} batch_confidences = {} target_columns = self.model_metadata['target_columns'] for target in target_columns: model = self.models[target] preds = model.predict(batch_df) probas = model.predict_proba(batch_df) confs = np.max(probas, axis=1) batch_predictions[target] = preds batch_confidences[target] = confs # Process results with enhanced logic for i, anomaly_data in enumerate(anomaly_list): # Get base predictions predictions = {target: int(batch_predictions[target][i]) for target in target_columns} confidences = {target: float(batch_confidences[target][i]) for target in target_columns} enhanced_features = all_features[i] # Apply safety override rules if enabled if apply_safety_rules and self.safety_rules: predictions = self._apply_safety_override_rules(enhanced_features, predictions) total_criticality = sum(predictions.values()) overall_confidence = np.mean(list(confidences.values())) # Enhanced business logic needs_review = self._determine_manual_review_need( enhanced_features, predictions, overall_confidence, confidence_threshold ) equipment_risk_assessment = self._assess_equipment_risk(enhanced_features, predictions) # Find similar anomalies (optional for batch) similar_anomalies = [] if include_similar and self.sentence_model is not None: similar_anomalies = self._find_similar_anomalies( anomaly_data.get('Description', ''), top_k=2 ) # Format response if format_type == 'simple': result = self._format_simple_response( anomaly_data, predictions, total_criticality, overall_confidence, needs_review, equipment_risk_assessment ) else: result = self._format_rich_response( anomaly_data, predictions, confidences, total_criticality, overall_confidence, similar_anomalies, needs_review, confidence_threshold, equipment_risk_assessment, enhanced_features ) results.append(result) processing_time = time.time() - start_time print(f"✓ Enhanced batch processing completed in {processing_time:.2f}s") print(f" Average time per anomaly: {processing_time/len(anomaly_list):.3f}s") flagged_count = sum(1 for r in results if r.get('needs_manual_review', False)) safety_overrides = sum(1 for r in results if r.get('safety_override_applied', False)) print(f" Flagged for manual review: {flagged_count}/{len(anomaly_list)} ({flagged_count/len(anomaly_list)*100:.1f}%)") print(f" Safety overrides applied: {safety_overrides}/{len(anomaly_list)} ({safety_overrides/len(anomaly_list)*100:.1f}%)") return results except Exception as e: # Return error for all items in batch error_result = { 'error': f'Enhanced batch prediction failed: {str(e)}', 'timestamp': datetime.now().isoformat() } return [error_result] * len(anomaly_list) def _extract_enhanced_features_single(self, anomaly_data: Dict) -> Dict: """Extract enhanced features including equipment intelligence""" # Create temporary DataFrame for feature engineering temp_df = pd.DataFrame([anomaly_data]) # Apply enhanced feature engineering (matching training pipeline) enhanced_features = self._extract_enhanced_features(temp_df) # Prepare feature dict with all required features feature_columns = self.model_metadata.get('all_feature_columns', []) input_data = {} # Text feature input_data['Description'] = anomaly_data.get('Description', '') # Enhanced numerical features numerical_features = self.model_metadata.get('numerical_features', []) for feat in numerical_features: if feat in enhanced_features.columns: value = enhanced_features[feat].iloc[0] # Ensure proper type conversion if pd.isna(value): input_data[feat] = 0.0 elif isinstance(value, (bool, np.bool_)): input_data[feat] = float(value) else: input_data[feat] = float(value) else: input_data[feat] = 0.0 # Categorical features categorical_features = self.model_metadata.get('categorical_features', []) for feat in categorical_features: input_data[feat] = anomaly_data.get(feat, 'Unknown') return input_data def _extract_enhanced_features(self, df): """Extract enhanced features (matching training pipeline logic)""" import re features_df = df.copy() # Create combined text field features_df['combined_text'] = features_df['Description'].fillna('') + ' ' + features_df.get('Description de l\'équipement', '').fillna('') features_df['combined_text_lower'] = features_df['combined_text'].str.lower() # Basic text features features_df['description_length'] = features_df['Description'].str.len() features_df['description_word_count'] = features_df['Description'].str.split().str.len() features_df['equipment_desc_length'] = features_df.get('Description de l\'équipement', '').str.len() features_df['equipment_desc_word_count'] = features_df.get('Description de l\'équipement', '').str.split().str.len() features_df['combined_length'] = features_df['combined_text'].str.len() features_df['combined_word_count'] = features_df['combined_text'].str.split().str.len() # Equipment intelligence classification def classify_equipment_type(equipment_desc): """Classify equipment based on training analysis""" equipment_upper = str(equipment_desc).upper() # Equipment type scoring (from training pipeline) if any(keyword in equipment_upper for keyword in ['ALTERNATEUR', 'TRANSFO PRINCIPAL', 'PROTECTION ALTERNATEUR']): return 'ELECTRICAL_CRITICAL', 8.0 elif any(keyword in equipment_upper for keyword in ['VENTILATEUR DE REFROIDISSEMENT', 'REFROIDISSEMENT TP', 'MOTEUR VENTILATEUR DE REFROIDISSEMENT']): return 'COOLING_CRITICAL', 7.5 elif any(keyword in equipment_upper for keyword in ['TURBINE', 'SOUPAPE REGULATRICE', 'REFRIGERANT HUILE', 'POMPE DE SOULÈVEMENT']): return 'TURBINE_SYSTEMS', 7.5 elif any(keyword in equipment_upper for keyword in ['DISJONCTEUR', 'TRANSFORMATEUR', 'MOTEUR', 'ARMOIRE', 'GROUPE']): return 'ELECTRICAL_STANDARD', 6.5 elif any(keyword in equipment_upper for keyword in ['RECHAUFFEUR', 'RÉCHAUFFEUR', 'CHAUDIERE', 'CHAUDIÈRE']): return 'HEATING_SYSTEMS', 6.5 elif any(keyword in equipment_upper for keyword in ['VENTILATEUR', 'TIRAGE', 'SOUFFLAGE', 'AIR PRIMAIRE', 'AIR SECONDAIRE']): return 'VENTILATION_SYSTEMS', 6.0 elif any(keyword in equipment_upper for keyword in ['POMPE', 'SOUPAPE', 'VANNE', 'CONVOYEUR', 'BROYEUR', 'COAL FEEDER']): return 'PROCESS_SYSTEMS', 5.5 elif any(keyword in equipment_upper for keyword in ['DECRASSEUR', 'DÉGRILLEUR', 'FILTRE', 'CAPTEUR', 'TRANSMETTEUR']): return 'AUXILIARY_SYSTEMS', 5.0 else: return 'UNKNOWN', 4.5 def detect_equipment_redundancy(equipment_desc): """Detect equipment redundancy based on naming patterns""" equipment_upper = str(equipment_desc).upper() if any(pattern in equipment_upper for pattern in ['PRINCIPAL', 'UNIQUE']): return 'SINGLE_CRITICAL', 1.3 elif any(re.search(pattern, equipment_upper) for pattern in [r'\b[AB]$', r'N°[12]$', r'PRIMAIRE$', r'SECONDAIRE$']): return 'DUAL_SYSTEM', 1.0 elif any(re.search(pattern, equipment_upper) for pattern in [r'N°[3-9]$', r'N°[0-9][0-9]$']): return 'MULTIPLE_SYSTEM', 0.8 else: return 'UNKNOWN_REDUNDANCY', 1.0 # Apply equipment intelligence if 'Description de l\'équipement' in features_df.columns: equipment_classifications = features_df['Description de l\'équipement'].apply(classify_equipment_type) features_df['equipment_type_class'] = [x[0] for x in equipment_classifications] features_df['equipment_base_criticality'] = [x[1] for x in equipment_classifications] redundancy_classifications = features_df['Description de l\'équipement'].apply(detect_equipment_redundancy) features_df['equipment_redundancy_class'] = [x[0] for x in redundancy_classifications] features_df['equipment_redundancy_multiplier'] = [x[1] for x in redundancy_classifications] else: features_df['equipment_type_class'] = 'UNKNOWN' features_df['equipment_base_criticality'] = 4.5 features_df['equipment_redundancy_class'] = 'UNKNOWN_REDUNDANCY' features_df['equipment_redundancy_multiplier'] = 1.0 # Section risk multiplier section_risk_multipliers = {'34EL': 1.2, '34MM': 1.1, '34MD': 1.1, '34MC': 1.0, '34CT': 1.0} features_df['section_risk_multiplier'] = features_df.get('Section propriétaire', '').map(section_risk_multipliers).fillna(1.0) # Combined equipment risk score features_df['equipment_risk_score'] = (features_df['equipment_base_criticality'] * features_df['equipment_redundancy_multiplier'] * features_df['section_risk_multiplier']) # Enhanced keyword extraction def extract_keywords_dual_field(description, equipment_desc, keyword_dict): """Extract keywords from both description and equipment description""" combined_text = (str(description) + ' ' + str(equipment_desc)).lower() found_keywords = [] for category, keywords in keyword_dict.items(): for keyword in keywords: if keyword in combined_text: found_keywords.append(category) break return found_keywords # Keyword dictionaries (from training pipeline) equipment_keywords = { 'pompe': ['pompe', 'pompes'], 'vanne': ['vanne', 'vannes'], 'ventilateur': ['ventilateur', 'ventilateurs', 'ventilo'], 'moteur': ['moteur', 'moteurs', 'moto'], 'alternateur': ['alternateur', 'alternateurs'], 'transformateur': ['transformateur', 'transformateurs', 'transfo'], 'turbine': ['turbine', 'turbines'], 'principal': ['principal', 'principale'], 'groupe': ['groupe', 'groupes'] } problem_keywords = { 'fuite': ['fuite', 'fuites', 'fuit', 'fuyant'], 'vibration': ['vibration', 'vibrations', 'vibre'], 'bruit_anormal': ['bruit anormal', 'bruit anormale'], 'percement': ['percement', 'percé', 'percée'], 'éclatement': ['éclatement', 'eclatement'], 'fissure': ['fissure', 'fissuré', 'fissures'], 'aggravation': ['aggravation'], 'sifflement': ['sifflement', 'siffler'], 'défaillance': ['défaillance', 'défaillant'], 'dysfonctionnement': ['dysfonctionnement', 'dysfonctionnel'], 'sens_inverse': ['sens inverse', 'sens contraire'], 'surchauffe': ['surchauffe', 'surchauffé', 'température élevée', 'temp elevee'] } action_keywords = { 'maintenance': ['maintenance', 'entretien'], 'prévision': ['prévoir', 'prévoire', 'prevoir'], 'remplacement': ['remplacement', 'remplacer', 'remplacé'] } urgency_keywords = { 'safety': ['safety', 'sécurité'], 'urgent': ['urgent', 'urgence'], 'critique': ['critique', 'critiques'], 'important': ['important', 'importante'] } # Apply keyword extraction description_col = features_df['Description'] equipment_col = features_df.get('Description de l\'équipement', '') features_df['equipment_mentioned'] = features_df.apply( lambda row: extract_keywords_dual_field(row['Description'], row.get('Description de l\'équipement', ''), equipment_keywords), axis=1 ) features_df['equipment_count'] = features_df['equipment_mentioned'].str.len() features_df['problem_types'] = features_df.apply( lambda row: extract_keywords_dual_field(row['Description'], row.get('Description de l\'équipement', ''), problem_keywords), axis=1 ) features_df['problem_count'] = features_df['problem_types'].str.len() features_df['actions_mentioned'] = features_df.apply( lambda row: extract_keywords_dual_field(row['Description'], row.get('Description de l\'équipement', ''), action_keywords), axis=1 ) features_df['action_count'] = features_df['actions_mentioned'].str.len() features_df['urgency_indicators'] = features_df.apply( lambda row: extract_keywords_dual_field(row['Description'], row.get('Description de l\'équipement', ''), urgency_keywords), axis=1 ) features_df['has_urgency'] = (features_df['urgency_indicators'].str.len() > 0).astype(int) # Critical failure pattern detection features_df['has_structural_failure'] = features_df['combined_text_lower'].str.contains( 'percement|éclatement|eclatement|fissure|rupture', regex=True, na=False ).astype(int) features_df['has_equipment_malfunction'] = features_df['combined_text_lower'].str.contains( 'sens inverse|dysfonctionnement|défaillance|défaut|panne', regex=True, na=False ).astype(int) features_df['has_escalation'] = features_df['combined_text_lower'].str.contains( 'aggravation|empiré|empire', regex=True, na=False ).astype(int) features_df['has_safety_mention'] = features_df['Description'].str.contains('SAFETY', case=False, na=False).astype(int) # Specific high-risk combinations features_df['electrical_cooling_issue'] = ( (features_df['equipment_type_class'].isin(['ELECTRICAL_CRITICAL', 'ELECTRICAL_STANDARD'])) & (features_df['combined_text_lower'].str.contains('refroidissement|ventilateur|température', regex=True, na=False)) ).astype(int) features_df['turbine_oil_issue'] = ( (features_df['equipment_type_class'] == 'TURBINE_SYSTEMS') & (features_df['combined_text_lower'].str.contains('huile|fuite|graissage', regex=True, na=False)) ).astype(int) features_df['main_equipment_failure'] = ( (features_df['equipment_redundancy_class'] == 'SINGLE_CRITICAL') & (features_df['has_structural_failure'] == 1) ).astype(int) # Enhanced compound features features_df['fuite_vapeur'] = features_df['combined_text_lower'].str.contains('fuite.*vapeur|vapeur.*fuite', regex=True, na=False).astype(int) features_df['fuite_huile'] = features_df['combined_text_lower'].str.contains('fuite.*huile|huile.*fuite', regex=True, na=False).astype(int) features_df['fuite_eau'] = features_df['combined_text_lower'].str.contains('fuite.*eau|eau.*fuite', regex=True, na=False).astype(int) features_df['bruit_anormal'] = features_df['combined_text_lower'].str.contains('bruit anormal', regex=True, na=False).astype(int) features_df['vibration_excessive'] = features_df['combined_text_lower'].str.contains('vibration.*excessive|vibration.*élevée', regex=True, na=False).astype(int) features_df['temperature_elevee'] = features_df['combined_text_lower'].str.contains('température élevée|temp élevée|temp elevee', regex=True, na=False).astype(int) features_df['maintenance_planning'] = features_df['combined_text_lower'].str.contains('prévoir|prévoire|planifier', regex=True, na=False).astype(int) features_df['is_recurring'] = features_df['combined_text_lower'].str.contains('fréquent|répétitif|souvent', regex=True, na=False).astype(int) # Technical features features_df['has_measurements'] = features_df['combined_text_lower'].str.contains(r'\d+\s*°c|\d+\s*bar|\d+\s*%', regex=True, na=False).astype(int) features_df['has_equipment_codes'] = features_df['combined_text_lower'].str.contains(r'[A-Z0-9]{5,}', regex=True, na=False).astype(int) features_df['has_location_details'] = features_df['combined_text_lower'].str.contains('niveau|angle|côté|palier', regex=True, na=False).astype(int) # Enhanced severity scoring severity_words = { 'critique': 4, 'grave': 4, 'majeur': 4, 'important': 3, 'total': 5, 'complet': 5, 'rupture': 5, 'éclatement': 5, 'percement': 5, 'fissure': 4, 'aggravation': 4, 'urgent': 3 } def calculate_enhanced_severity_score(text): text = str(text).lower() max_score = 0 for word, weight in severity_words.items(): if word in text: max_score = max(max_score, weight) return max_score features_df['enhanced_severity_score'] = features_df['combined_text_lower'].apply(calculate_enhanced_severity_score) # Equipment-Problem Risk Matrix def calculate_equipment_problem_risk(equipment_type, problem_types, has_structural): base_risk = 1.0 if equipment_type in ['ELECTRICAL_CRITICAL', 'TURBINE_SYSTEMS', 'COOLING_CRITICAL']: base_risk = 1.5 elif equipment_type in ['ELECTRICAL_STANDARD', 'HEATING_SYSTEMS']: base_risk = 1.2 if has_structural: base_risk *= 2.0 if 'vibration' in problem_types: base_risk *= 1.3 if 'fuite' in problem_types: base_risk *= 1.2 return min(base_risk, 3.0) features_df['equipment_problem_risk'] = features_df.apply( lambda row: calculate_equipment_problem_risk( row['equipment_type_class'], row['problem_types'], row['has_structural_failure'] ), axis=1 ) # Technical complexity features_df['technical_complexity'] = ( features_df['combined_word_count'] / 15 + features_df['equipment_count'] + features_df['problem_count'] + features_df['has_measurements'] + features_df['has_equipment_codes'] + features_df['has_location_details'] ) # Fill missing values and ensure proper types numeric_columns = features_df.select_dtypes(include=[np.number]).columns features_df[numeric_columns] = features_df[numeric_columns].fillna(0) for col in features_df.select_dtypes(include=[np.integer, np.floating, bool]).columns: features_df[col] = pd.to_numeric(features_df[col], errors='coerce').fillna(0) return features_df def _predict_criticality(self, input_data: Dict) -> tuple: """Make criticality predictions using enhanced models""" # Convert to DataFrame input_df = pd.DataFrame([input_data]) target_columns = self.model_metadata['target_columns'] predictions = {} confidences = {} probabilities = {} for target in target_columns: model = self.models[target] pred = model.predict(input_df)[0] pred_proba = model.predict_proba(input_df)[0] confidence = np.max(pred_proba) predictions[target] = int(pred) confidences[target] = float(confidence) probabilities[target] = [float(x) for x in pred_proba] return predictions, confidences, probabilities def _apply_safety_override_rules(self, enhanced_features: Dict, predictions: Dict) -> Dict: """Apply safety override rules to predictions""" def _apply_safety_override_rules(self, enhanced_features: Dict, predictions: Dict) -> Dict: """Apply safety override rules to predictions""" if not self.safety_rules: return predictions modified_predictions = predictions.copy() safety_override_applied = False # Rule 1: Structural failure override if enhanced_features.get('has_structural_failure', 0) == 1: # Ensure minimum criticality of 9 for structural failures total_current = sum(modified_predictions.values()) if total_current < 9: # Boost Process Safety to 5 first (most critical for structural failures) if modified_predictions['Process Safety'] < 5: modified_predictions['Process Safety'] = 5 safety_override_applied = True # Then boost Fiabilité if still needed total_after_safety = sum(modified_predictions.values()) if total_after_safety < 9: needed_boost = 9 - total_after_safety new_fiabilite = min(5, modified_predictions['Fiabilité Intégrité'] + needed_boost) modified_predictions['Fiabilité Intégrité'] = new_fiabilite safety_override_applied = True # Rule 2: Cooling critical equipment override if enhanced_features.get('equipment_type_class', '') == 'COOLING_CRITICAL': # Ensure minimum criticality of 10 for cooling critical equipment total_current = sum(modified_predictions.values()) if total_current < 10: # Boost all components proportionally needed_boost = 10 - total_current for component in modified_predictions: if modified_predictions[component] < 5: boost = min(2, needed_boost // 3 + 1) modified_predictions[component] = min(5, modified_predictions[component] + boost) needed_boost -= boost safety_override_applied = True if needed_boost <= 0: break # Rule 3: Safety mention boost if enhanced_features.get('has_safety_mention', 0) == 1: # Add +2 to Process Safety for safety mentions if modified_predictions['Process Safety'] < 5: boost = min(2, 5 - modified_predictions['Process Safety']) modified_predictions['Process Safety'] += boost safety_override_applied = True # Rule 4: Turbine oil issue override if enhanced_features.get('turbine_oil_issue', 0) == 1: # Ensure minimum criticality of 8 for turbine oil issues total_current = sum(modified_predictions.values()) if total_current < 8: # Boost Fiabilité and Disponibilité (oil issues affect both) needed_boost = 8 - total_current for component in ['Fiabilité Intégrité', 'Disponibilté']: if needed_boost > 0 and modified_predictions[component] < 4: boost = min(2, needed_boost) modified_predictions[component] = min(5, modified_predictions[component] + boost) needed_boost -= boost safety_override_applied = True # Rule 5: Electrical critical equipment override if enhanced_features.get('equipment_type_class', '') == 'ELECTRICAL_CRITICAL': # Conservative boost for electrical critical equipment for component in modified_predictions: if modified_predictions[component] >= 3: # Only boost already elevated predictions boost = min(1, 5 - modified_predictions[component]) if boost > 0: modified_predictions[component] += boost safety_override_applied = True return modified_predictions def _determine_manual_review_need(self, enhanced_features: Dict, predictions: Dict, overall_confidence: float, confidence_threshold: float) -> bool: """Enhanced logic to determine if manual review is needed""" # Base confidence check if overall_confidence < confidence_threshold: return True # Critical equipment always needs review for high predictions if enhanced_features.get('equipment_type_class', '') in ['ELECTRICAL_CRITICAL', 'COOLING_CRITICAL', 'TURBINE_SYSTEMS']: if sum(predictions.values()) >= 8: return True # Structural failures always need review if enhanced_features.get('has_structural_failure', 0) == 1: return True # Safety mentions need review if enhanced_features.get('has_safety_mention', 0) == 1: return True # High criticality cases need review if sum(predictions.values()) >= 10: return True # Equipment malfunction with high-risk equipment if (enhanced_features.get('has_equipment_malfunction', 0) == 1 and enhanced_features.get('equipment_type_class', '') in ['ELECTRICAL_CRITICAL', 'TURBINE_SYSTEMS']): return True return False def _assess_equipment_risk(self, enhanced_features: Dict, predictions: Dict) -> Dict: """Assess equipment-specific risk factors""" equipment_type = enhanced_features.get('equipment_type_class', 'UNKNOWN') total_criticality = sum(predictions.values()) risk_assessment = { 'equipment_type': equipment_type, 'redundancy_class': enhanced_features.get('equipment_redundancy_class', 'UNKNOWN'), 'base_risk_score': enhanced_features.get('equipment_risk_score', 4.5), 'risk_level': 'LOW', 'risk_factors': [], 'business_impact': 'MINOR' } # Determine risk level based on equipment type and criticality if equipment_type == 'COOLING_CRITICAL': risk_assessment['risk_level'] = 'CRITICAL' risk_assessment['business_impact'] = 'SEVERE' risk_assessment['risk_factors'].append('Critical cooling system failure') elif equipment_type == 'ELECTRICAL_CRITICAL': if total_criticality >= 8: risk_assessment['risk_level'] = 'HIGH' risk_assessment['business_impact'] = 'MAJOR' else: risk_assessment['risk_level'] = 'MEDIUM' risk_assessment['business_impact'] = 'MODERATE' risk_assessment['risk_factors'].append('Electrical critical infrastructure') elif equipment_type == 'TURBINE_SYSTEMS': if total_criticality >= 8: risk_assessment['risk_level'] = 'HIGH' risk_assessment['business_impact'] = 'MAJOR' else: risk_assessment['risk_level'] = 'MEDIUM' risk_assessment['business_impact'] = 'MODERATE' risk_assessment['risk_factors'].append('Turbine system component') # Add specific risk factors if enhanced_features.get('has_structural_failure', 0) == 1: risk_assessment['risk_factors'].append('Structural integrity compromise') risk_assessment['risk_level'] = 'HIGH' if enhanced_features.get('has_safety_mention', 0) == 1: risk_assessment['risk_factors'].append('Safety concern flagged') if enhanced_features.get('equipment_redundancy_class', '') == 'SINGLE_CRITICAL': risk_assessment['risk_factors'].append('Single point of failure') if enhanced_features.get('turbine_oil_issue', 0) == 1: risk_assessment['risk_factors'].append('Turbine lubrication system issue') if enhanced_features.get('electrical_cooling_issue', 0) == 1: risk_assessment['risk_factors'].append('Electrical equipment cooling problem') # Determine business impact based on total criticality and equipment type if total_criticality >= 12: risk_assessment['business_impact'] = 'SEVERE' elif total_criticality >= 10: risk_assessment['business_impact'] = 'MAJOR' elif total_criticality >= 8: risk_assessment['business_impact'] = 'MODERATE' return risk_assessment def _find_similar_anomalies(self, description: str, top_k: int = 3) -> List[Dict]: """Find similar historical anomalies""" if not description or self.sentence_model is None or self.embeddings is None: return [] try: # Encode new description new_embedding = self.sentence_model.encode([description]) # Calculate similarities similarities = cosine_similarity(new_embedding, self.embeddings)[0] # Get top k most similar top_indices = np.argsort(similarities)[::-1] similar_anomalies = [] for idx in top_indices[:top_k*2]: # Check more to filter similarity_score = float(similarities[idx]) # Skip if too similar (likely duplicate) or too dissimilar if similarity_score > 0.99 or similarity_score < 0.15: continue if len(similar_anomalies) >= top_k: break similar_anomalies.append({ 'description': self.embedding_metadata['descriptions'][idx], 'criticality': int(self.embedding_metadata['criticality_scores'][idx]), 'similarity_score': round(similarity_score, 3), 'section': self.embedding_metadata.get('sections', ['Unknown'])[idx], 'equipment_mentioned': self.embedding_metadata.get('equipment_mentioned', [[]])[idx] }) return similar_anomalies except Exception as e: print(f"Warning: Similarity search failed: {e}") return [] def _format_simple_response(self, anomaly_data: Dict, predictions: Dict, total_criticality: int, overall_confidence: float, needs_review: bool, equipment_risk_assessment: Dict) -> Dict: """Format simple response for database insertion""" return { 'timestamp': datetime.now().isoformat(), 'input_description': anomaly_data.get('Description', ''), 'input_section': anomaly_data.get('Section propriétaire', ''), 'input_equipment': anomaly_data.get('Description de l\'équipement', ''), # Predictions 'predicted_criticite': total_criticality, 'predicted_fiabilite': predictions['Fiabilité Intégrité'], 'predicted_disponibilite': predictions['Disponibilté'], 'predicted_safety': predictions['Process Safety'], # AI Metrics 'ai_confidence': round(overall_confidence, 3), 'needs_manual_review': bool(needs_review), # Equipment Intelligence 'equipment_type': equipment_risk_assessment['equipment_type'], 'equipment_risk_level': equipment_risk_assessment['risk_level'], 'business_impact': equipment_risk_assessment['business_impact'], 'safety_override_applied': any(pred > 3 for pred in predictions.values()), # Metadata 'model_version': '2.0_enhanced', 'processing_timestamp': datetime.now().isoformat() } def _format_rich_response(self, anomaly_data: Dict, predictions: Dict, confidences: Dict, total_criticality: int, overall_confidence: float, similar_anomalies: List, needs_review: bool, confidence_threshold: float, equipment_risk_assessment: Dict, enhanced_features: Dict) -> Dict: """Format rich response for UI display""" # Calculate additional metrics reliability_score = self._calculate_reliability_score( confidences, enhanced_features, equipment_risk_assessment ) return { 'timestamp': datetime.now().isoformat(), 'input_description': anomaly_data.get('Description', ''), 'input_section': anomaly_data.get('Section propriétaire', ''), 'input_equipment': anomaly_data.get('Description de l\'équipement', ''), 'predictions': { 'criticite_totale': total_criticality, 'components': { 'fiabilite_integrite': predictions['Fiabilité Intégrité'], 'disponibilite': predictions['Disponibilté'], 'process_safety': predictions['Process Safety'] } }, 'confidence': { 'overall_confidence': round(overall_confidence, 3), 'reliability_score': round(reliability_score, 3), 'component_confidence': { 'fiabilite_integrite': round(confidences['Fiabilité Intégrité'], 3), 'disponibilite': round(confidences['Disponibilté'], 3), 'process_safety': round(confidences['Process Safety'], 3) }, 'needs_manual_review': bool(needs_review), 'confidence_threshold': confidence_threshold, 'recommendation': self._get_confidence_recommendation(reliability_score) }, 'equipment_intelligence': { 'equipment_type': equipment_risk_assessment['equipment_type'], 'redundancy_class': equipment_risk_assessment['redundancy_class'], 'risk_level': equipment_risk_assessment['risk_level'], 'business_impact': equipment_risk_assessment['business_impact'], 'risk_factors': equipment_risk_assessment['risk_factors'], 'base_risk_score': round(equipment_risk_assessment['base_risk_score'], 2) }, 'safety_analysis': { 'structural_failure_detected': bool(enhanced_features.get('has_structural_failure', 0)), 'safety_mention_present': bool(enhanced_features.get('has_safety_mention', 0)), 'equipment_malfunction_detected': bool(enhanced_features.get('has_equipment_malfunction', 0)), 'escalation_detected': bool(enhanced_features.get('has_escalation', 0)), 'safety_override_applied': any(pred > 3 for pred in predictions.values()), 'urgency_level': self._determine_urgency_level(total_criticality, reliability_score, equipment_risk_assessment) }, 'similar_anomalies': similar_anomalies, 'analysis': { 'problem_types_detected': enhanced_features.get('problem_types', []), 'equipment_mentioned': enhanced_features.get('equipment_mentioned', []), 'severity_score': enhanced_features.get('enhanced_severity_score', 0), 'technical_complexity': round(enhanced_features.get('technical_complexity', 0), 2), 'pattern_indicators': self._identify_critical_patterns(enhanced_features) }, 'model_metadata': { 'version': '2.0_enhanced', 'features_used': len([k for k in enhanced_features.keys() if k != 'Description']), 'equipment_intelligence_enabled': True, 'safety_rules_enabled': bool(self.safety_rules) } } def _calculate_reliability_score(self, confidences: Dict, enhanced_features: Dict, equipment_risk_assessment: Dict) -> float: """Calculate enhanced reliability score""" # Base prediction confidence prediction_confidence = np.mean(list(confidences.values())) # Model agreement (lower std = higher agreement) model_agreement = 1.0 - (np.std(list(confidences.values())) / max(np.mean(list(confidences.values())), 0.1)) # Feature completeness has_description = len(enhanced_features.get('Description', '')) > 10 has_equipment = enhanced_features.get('equipment_type_class', 'UNKNOWN') != 'UNKNOWN' has_section = enhanced_features.get('Section propriétaire', 'Unknown') != 'Unknown' feature_completeness = (has_description + has_equipment + has_section) / 3 # Equipment intelligence confidence boost equipment_confidence_boost = 0.0 if equipment_risk_assessment['equipment_type'] != 'UNKNOWN': equipment_confidence_boost = 0.1 # Pattern detection confidence pattern_confidence = 0.0 if enhanced_features.get('has_safety_mention', 0) == 1: pattern_confidence += 0.1 if enhanced_features.get('has_structural_failure', 0) == 1: pattern_confidence += 0.15 if enhanced_features.get('equipment_problem_risk', 0) > 1.5: pattern_confidence += 0.1 # Combine all factors reliability_score = ( prediction_confidence * 0.4 + model_agreement * 0.25 + feature_completeness * 0.2 + equipment_confidence_boost + pattern_confidence ) return min(reliability_score, 1.0) def _get_confidence_recommendation(self, reliability_score: float) -> str: """Get confidence-based recommendation""" if reliability_score >= 0.85: return "Very high confidence - Prediction highly reliable" elif reliability_score >= 0.75: return "High confidence - Prediction can be trusted" elif reliability_score >= 0.65: return "Medium confidence - Consider expert review for critical decisions" elif reliability_score >= 0.5: return "Low confidence - Manual review recommended" else: return "Very low confidence - Expert assessment required" def _determine_urgency_level(self, total_criticality: int, reliability_score: float, equipment_risk_assessment: Dict) -> str: """Determine enhanced urgency level""" # Adjust criticality by reliability and equipment risk adjusted_criticality = total_criticality * reliability_score # Equipment type urgency multiplier equipment_urgency_multiplier = 1.0 if equipment_risk_assessment['equipment_type'] in ['COOLING_CRITICAL', 'ELECTRICAL_CRITICAL']: equipment_urgency_multiplier = 1.3 elif equipment_risk_assessment['equipment_type'] in ['TURBINE_SYSTEMS']: equipment_urgency_multiplier = 1.2 final_urgency_score = adjusted_criticality * equipment_urgency_multiplier if final_urgency_score >= 14: return "EMERGENCY - Immediate shutdown may be required" elif final_urgency_score >= 12: return "CRITICAL - Immediate action required (within 1 hour)" elif final_urgency_score >= 9: return "HIGH - Action required within 24 hours" elif final_urgency_score >= 6: return "MEDIUM - Action required within 1 week" else: return "LOW - Routine maintenance scheduling" def _identify_critical_patterns(self, enhanced_features: Dict) -> List[str]: """Identify critical patterns in the anomaly""" patterns = [] if enhanced_features.get('has_structural_failure', 0) == 1: patterns.append('Structural failure detected') if enhanced_features.get('has_safety_mention', 0) == 1: patterns.append('Safety concern explicitly mentioned') if enhanced_features.get('electrical_cooling_issue', 0) == 1: patterns.append('Electrical equipment cooling issue') if enhanced_features.get('turbine_oil_issue', 0) == 1: patterns.append('Turbine lubrication system problem') if enhanced_features.get('main_equipment_failure', 0) == 1: patterns.append('Critical single-point equipment failure') if enhanced_features.get('has_escalation', 0) == 1: patterns.append('Problem escalation indicated') if enhanced_features.get('vibration_excessive', 0) == 1: patterns.append('Excessive vibration detected') if enhanced_features.get('temperature_elevee', 0) == 1: patterns.append('High temperature condition') if enhanced_features.get('enhanced_severity_score', 0) >= 4: patterns.append('High severity language detected') return patterns # ============== CONVENIENCE FUNCTIONS ============== # Global instance for easy use _enhanced_ai_instance = None def get_enhanced_ai_instance(): """Get singleton enhanced AI instance""" global _enhanced_ai_instance if _enhanced_ai_instance is None: _enhanced_ai_instance = EnhancedAnomalyIntelligence() return _enhanced_ai_instance def predict_anomaly_single_enhanced(anomaly_data: Dict, **kwargs) -> Dict: """Convenience function for enhanced single prediction""" ai = get_enhanced_ai_instance() return ai.predict_single(anomaly_data, **kwargs) def predict_anomaly_batch_enhanced(anomaly_list: List[Dict], **kwargs) -> List[Dict]: """Convenience function for enhanced batch prediction""" ai = get_enhanced_ai_instance() return ai.predict_batch(anomaly_list, **kwargs) def process_excel_upload_enhanced(excel_data: pd.DataFrame, confidence_threshold: float = 0.7) -> pd.DataFrame: """ Process Excel upload with enhanced AI predictions Args: excel_data: DataFrame from uploaded Excel confidence_threshold: Confidence threshold for manual review Returns: DataFrame with enhanced AI prediction columns """ # Convert DataFrame to list of dicts anomaly_list = excel_data.to_dict('records') # Get enhanced batch predictions predictions = predict_anomaly_batch_enhanced( anomaly_list, confidence_threshold=confidence_threshold, include_similar=False, # Skip for batch processing speed format_type='simple', apply_safety_rules=True ) # Add enhanced prediction columns to original DataFrame result_df = excel_data.copy() # Enhanced AI prediction columns result_df['AI_Predicted_Criticite'] = [p.get('predicted_criticite', 0) for p in predictions] result_df['AI_Predicted_Fiabilite'] = [p.get('predicted_fiabilite', 0) for p in predictions] result_df['AI_Predicted_Disponibilite'] = [p.get('predicted_disponibilite', 0) for p in predictions] result_df['AI_Predicted_Safety'] = [p.get('predicted_safety', 0) for p in predictions] result_df['AI_Confidence'] = [p.get('ai_confidence', 0.0) for p in predictions] result_df['AI_Needs_Review'] = [bool(p.get('needs_manual_review', True)) for p in predictions] # Equipment intelligence columns result_df['AI_Equipment_Type'] = [p.get('equipment_type', 'UNKNOWN') for p in predictions] result_df['AI_Risk_Level'] = [p.get('equipment_risk_level', 'LOW') for p in predictions] result_df['AI_Business_Impact'] = [p.get('business_impact', 'MINOR') for p in predictions] result_df['AI_Safety_Override'] = [bool(p.get('safety_override_applied', False)) for p in predictions] # Human verification columns result_df['Human_Verified'] = False result_df['Human_Criticite'] = None result_df['Human_Fiabilite'] = None result_df['Human_Disponibilite'] = None result_df['Human_Safety'] = None result_df['Correction_Reason'] = '' result_df['Verified_At'] = None result_df['Verified_By'] = '' result_df['Expert_Notes'] = '' return result_df # ============== ENHANCED EXAMPLE USAGE ============== if __name__ == "__main__": # Example 1: Enhanced single anomaly prediction print("="*70) print("TESTING ENHANCED SINGLE ANOMALY PREDICTION") print("="*70) single_anomaly = { 'Description': 'SAFETY : fuite vapeur importante sur TRANSFO PRINCIPAL, température élevée detectée, vibration excessive', 'Section propriétaire': '34EL', 'Description de l\'équipement': 'TRANSFO PRINCIPAL' } result = predict_anomaly_single_enhanced( single_anomaly, format_type='rich', apply_safety_rules=True, include_similar=True ) print("Enhanced rich format result:") print(f"Predicted Criticality: {result['predictions']['criticite_totale']}") print(f"Equipment Type: {result['equipment_intelligence']['equipment_type']}") print(f"Risk Level: {result['equipment_intelligence']['risk_level']}") print(f"Business Impact: {result['equipment_intelligence']['business_impact']}") print(f"Safety Override Applied: {result['safety_analysis']['safety_override_applied']}") print(f"Urgency Level: {result['safety_analysis']['urgency_level']}") print(f"Risk Factors: {result['equipment_intelligence']['risk_factors']}") # Example 2: Enhanced batch processing print("\n" + "="*70) print("TESTING ENHANCED BATCH PREDICTION") print("="*70) batch_anomalies = [ { 'Description': 'vibration excessive ALTERNATEUR, bruit anormal détecté', 'Section propriétaire': '34EL', 'Description de l\'équipement': 'ALTERNATEUR' }, { 'Description': 'fuite huile système hydraulique TURBINE, pression basse', 'Section propriétaire': '34MM', 'Description de l\'équipement': 'TURBINE' }, { 'Description': 'maintenance préventive DECRASSEUR à prévoir', 'Section propriétaire': '34MC', 'Description de l\'équipement': 'DECRASSEUR' }, { 'Description': 'percement conduite vapeur VENTILATEUR DE REFROIDISSEMENT TP', 'Section propriétaire': '34EL', 'Description de l\'équipement': 'VENTILATEUR DE REFROIDISSEMENT TP' } ] batch_results = predict_anomaly_batch_enhanced( batch_anomalies, confidence_threshold=0.7, format_type='simple', apply_safety_rules=True ) print("Enhanced batch results:") for i, result in enumerate(batch_results): print(f"\nAnomaly {i+1}:") print(f" Equipment Type: {result.get('equipment_type', 'N/A')}") print(f" Criticité: {result.get('predicted_criticite', 'N/A')}") print(f" Risk Level: {result.get('equipment_risk_level', 'N/A')}") print(f" Business Impact: {result.get('business_impact', 'N/A')}") print(f" Confidence: {result.get('ai_confidence', 'N/A')}") print(f" Safety Override: {result.get('safety_override_applied', 'N/A')}") print(f" Needs Review: {result.get('needs_manual_review', 'N/A')}") # Example 3: Enhanced Excel processing simulation print("\n" + "="*70) print("TESTING ENHANCED EXCEL PROCESSING") print("="*70) # Simulate Excel data with various equipment types excel_df = pd.DataFrame([ { 'Description': 'problème refroidissement TRANSFO PRINCIPAL', 'Section propriétaire': '34EL', 'Description de l\'équipement': 'TRANSFO PRINCIPAL', 'Date de détéction de l\'anomalie': '2025-01-15' }, { 'Description': 'SAFETY : éclatement tube chaudière, fissure détectée', 'Section propriétaire': '34MD', 'Description de l\'équipement': 'CHAUDIERE', 'Date de détéction de l\'anomalie': '2025-01-16' }, { 'Description': 'maintenance POMPE A prévoir', 'Section propriétaire': '34MC', 'Description de l\'équipement': 'POMPE', 'Date de détéction de l\'anomalie': '2025-01-17' } ]) processed_df = process_excel_upload_enhanced(excel_df, confidence_threshold=0.7) print("Enhanced processed Excel columns:") enhanced_columns = [col for col in processed_df.columns if col.startswith('AI_')] print(enhanced_columns) print("\nSample of enhanced processed data:") display_cols = ['Description', 'AI_Predicted_Criticite', 'AI_Equipment_Type', 'AI_Risk_Level', 'AI_Business_Impact', 'AI_Safety_Override', 'AI_Needs_Review'] print(processed_df[display_cols].to_string(index=False)) print("\n" + "🎯" + "="*68) print("ENHANCED ANOMALY INTELLIGENCE v2.0 TESTS COMPLETED SUCCESSFULLY!") print("="*70) print("✓ Equipment Intelligence Integration") print("✓ Safety Override Rules") print("✓ Enhanced Risk Assessment") print("✓ Conservative Prediction Bias") print("✓ Business Impact Analysis") print("✓ Production-Ready Performance") print("="*70)