Update sozo_gen.py

sozo_gen.py

@@ -269,11 +269,539 @@ def sanitize_for_firebase_key(text: str) -> str:
 
 # REPLACE THE OLD generate_report_draft WITH THIS CORRECTED VERSION
 def generate_report_draft(buf, name: str, ctx: str, uid: str, project_id: str, bucket):
-    logging.info(f"Generating report draft for project {project_id}")
+    """
+    Enhanced autonomous data analysis function that intelligently analyzes any dataset
+    and generates comprehensive, domain-appropriate reports with contextual visualizations.
+    
+    Maintains backward compatibility with existing function signature and outputs.
+    """
+    logging.info(f"Generating enhanced autonomous report draft for project {project_id}")
+    
+    # Load data safely (existing functionality preserved)
     df = load_dataframe_safely(buf, name)
+    
+    # Initialize LLM (existing setup preserved)
     llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", google_api_key=API_KEY, temperature=0.1)
+    
+    # Enhanced autonomous data analysis
+    try:
+        # Stage 1: Intelligent Data Classification and Deep Analysis
+        autonomous_context = perform_autonomous_data_analysis(df, ctx, name)
+        
+        # Stage 2: Generate Enhanced Report with Intelligent Narrative
+        enhanced_report = generate_intelligent_report(llm, autonomous_context)
+        
+        # Stage 3: Smart Chart Generation
+        chart_urls = generate_autonomous_charts(llm, df, enhanced_report, uid, project_id, bucket)
+        
+        # Preserve original output structure
+        return {"raw_md": enhanced_report, "chartUrls": chart_urls}
+        
+    except Exception as e:
+        logging.error(f"Enhanced analysis failed, falling back to original: {str(e)}")
+        # Fallback to original logic if enhancement fails
+        return generate_original_report(df, llm, ctx, uid, project_id, bucket)
+
+
+def perform_autonomous_data_analysis(df: pd.DataFrame, user_ctx: str, filename: str) -> Dict[str, Any]:
+    """
+    Performs comprehensive autonomous analysis of the dataset to understand its nature,
+    domain, and analytical potential.
+    """
+    logging.info("Performing autonomous data analysis...")
+    
+    # Basic data profiling
+    basic_info = {
+        "shape": df.shape,
+        "columns": list(df.columns),
+        "dtypes": df.dtypes.to_dict(),
+        "filename": filename,
+        "user_context": user_ctx
+    }
+    
+    # Intelligent domain classification
+    domain_analysis = classify_dataset_domain(df, filename)
+    
+    # Advanced statistical analysis
+    statistical_profile = generate_statistical_profile(df)
+    
+    # Relationship discovery
+    relationships = discover_data_relationships(df)
+    
+    # Temporal analysis if applicable
+    temporal_insights = analyze_temporal_patterns(df)
+    
+    # Data quality assessment
+    quality_metrics = assess_data_quality(df)
+    
+    # Business context inference
+    business_context = infer_business_context(df, domain_analysis)
+    
+    return {
+        "basic_info": basic_info,
+        "domain": domain_analysis,
+        "statistics": statistical_profile,
+        "relationships": relationships,
+        "temporal": temporal_insights,
+        "quality": quality_metrics,
+        "business_context": business_context,
+        "analysis_complexity": determine_analysis_complexity(df, domain_analysis)
+    }
+
+
+def classify_dataset_domain(df: pd.DataFrame, filename: str) -> Dict[str, Any]:
+    """
+    Intelligently classifies the dataset domain based on column patterns, data types,
+    and semantic analysis.
+    """
+    domain_indicators = {
+        "financial": ["amount", "price", "cost", "revenue", "profit", "transaction", "payment", "invoice"],
+        "survey": ["rating", "satisfaction", "response", "score", "survey", "feedback", "opinion"],
+        "scientific": ["measurement", "experiment", "test", "sample", "observation", "hypothesis", "variable"],
+        "marketing": ["campaign", "click", "conversion", "customer", "lead", "acquisition", "retention"],
+        "operational": ["process", "time", "duration", "status", "workflow", "performance", "efficiency"],
+        "sales": ["order", "product", "quantity", "sales", "customer", "deal", "pipeline"],
+        "hr": ["employee", "salary", "department", "performance", "training", "recruitment"],
+        "healthcare": ["patient", "diagnosis", "treatment", "medical", "health", "symptom", "medication"]
+    }
+    
+    # Analyze column names for domain indicators
+    columns_lower = [col.lower() for col in df.columns]
+    domain_scores = {}
+    
+    for domain, keywords in domain_indicators.items():
+        score = sum(1 for col in columns_lower for keyword in keywords if keyword in col)
+        domain_scores[domain] = score
+    
+    # Filename analysis
+    filename_lower = filename.lower()
+    for domain, keywords in domain_indicators.items():
+        if any(keyword in filename_lower for keyword in keywords):
+            domain_scores[domain] = domain_scores.get(domain, 0) + 2
+    
+    # Data type analysis
+    numeric_ratio = len(df.select_dtypes(include=[np.number]).columns) / len(df.columns)
+    categorical_ratio = len(df.select_dtypes(include=['object']).columns) / len(df.columns)
+    
+    # Determine primary domain
+    primary_domain = max(domain_scores, key=domain_scores.get) if domain_scores else "general"
+    
+    return {
+        "primary_domain": primary_domain,
+        "domain_confidence": domain_scores.get(primary_domain, 0),
+        "domain_scores": domain_scores,
+        "data_characteristics": {
+            "numeric_ratio": numeric_ratio,
+            "categorical_ratio": categorical_ratio,
+            "is_time_series": detect_time_series(df),
+            "is_transactional": detect_transactional_data(df),
+            "is_experimental": detect_experimental_data(df)
+        }
+    }
+
+
+def generate_statistical_profile(df: pd.DataFrame) -> Dict[str, Any]:
+    """
+    Generates comprehensive statistical profile of the dataset.
+    """
+    profile = {
+        "summary_stats": {},
+        "correlations": {},
+        "distributions": {},
+        "outliers": {},
+        "missing_data": {}
+    }
+    
+    # Summary statistics for numeric columns
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    if len(numeric_cols) > 0:
+        profile["summary_stats"] = df[numeric_cols].describe().to_dict()
+        
+        # Correlation analysis
+        if len(numeric_cols) > 1:
+            corr_matrix = df[numeric_cols].corr()
+            # Find strong correlations
+            strong_corrs = []
+            for i in range(len(corr_matrix.columns)):
+                for j in range(i+1, len(corr_matrix.columns)):
+                    corr_val = corr_matrix.iloc[i, j]
+                    if abs(corr_val) > 0.7:  # Strong correlation threshold
+                        strong_corrs.append({
+                            "var1": corr_matrix.columns[i],
+                            "var2": corr_matrix.columns[j],
+                            "correlation": corr_val
+                        })
+            profile["correlations"] = {"strong_correlations": strong_corrs}
+    
+    # Categorical analysis
+    categorical_cols = df.select_dtypes(include=['object']).columns
+    if len(categorical_cols) > 0:
+        profile["categorical_analysis"] = {}
+        for col in categorical_cols:
+            profile["categorical_analysis"][col] = {
+                "unique_count": df[col].nunique(),
+                "top_values": df[col].value_counts().head(5).to_dict()
+            }
+    
+    # Missing data analysis
+    missing_data = df.isnull().sum()
+    profile["missing_data"] = {
+        "columns_with_missing": missing_data[missing_data > 0].to_dict(),
+        "total_missing_percentage": (df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100
+    }
+    
+    return profile
+
+
+def discover_data_relationships(df: pd.DataFrame) -> Dict[str, Any]:
+    """
+    Discovers meaningful relationships and patterns in the data.
+    """
+    relationships = {
+        "key_relationships": [],
+        "patterns": [],
+        "anomalies": []
+    }
+    
+    # Identify potential key relationships
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    
+    if len(numeric_cols) > 1:
+        # Find interesting relationships
+        for col1 in numeric_cols:
+            for col2 in numeric_cols:
+                if col1 != col2:
+                    correlation = df[col1].corr(df[col2])
+                    if abs(correlation) > 0.5:  # Moderate to strong correlation
+                        relationships["key_relationships"].append({
+                            "variable1": col1,
+                            "variable2": col2,
+                            "relationship_strength": correlation,
+                            "relationship_type": "positive" if correlation > 0 else "negative"
+                        })
+    
+    # Identify patterns in categorical data
+    categorical_cols = df.select_dtypes(include=['object']).columns
+    for col in categorical_cols:
+        if df[col].nunique() < 20:  # Reasonable number of categories
+            value_counts = df[col].value_counts()
+            if len(value_counts) > 0:
+                relationships["patterns"].append({
+                    "column": col,
+                    "pattern_type": "categorical_distribution",
+                    "dominant_category": value_counts.index[0],
+                    "dominance_percentage": (value_counts.iloc[0] / len(df)) * 100
+                })
+    
+    return relationships
+
+
+def analyze_temporal_patterns(df: pd.DataFrame) -> Dict[str, Any]:
+    """
+    Analyzes temporal patterns if time-based columns are detected.
+    """
+    temporal_insights = {"has_temporal_data": False}
+    
+    # Detect date/time columns
+    date_columns = []
+    for col in df.columns:
+        if df[col].dtype == 'datetime64[ns]' or 'date' in col.lower() or 'time' in col.lower():
+            try:
+                pd.to_datetime(df[col])
+                date_columns.append(col)
+            except:
+                continue
+    
+    if date_columns:
+        temporal_insights["has_temporal_data"] = True
+        temporal_insights["date_columns"] = date_columns
+        
+        # Analyze temporal patterns for the first date column
+        primary_date_col = date_columns[0]
+        df_temp = df.copy()
+        df_temp[primary_date_col] = pd.to_datetime(df_temp[primary_date_col])
+        
+        temporal_insights["temporal_analysis"] = {
+            "date_range": {
+                "start": df_temp[primary_date_col].min().strftime('%Y-%m-%d'),
+                "end": df_temp[primary_date_col].max().strftime('%Y-%m-%d')
+            },
+            "time_span_days": (df_temp[primary_date_col].max() - df_temp[primary_date_col].min()).days,
+            "frequency": detect_temporal_frequency(df_temp[primary_date_col])
+        }
+    
+    return temporal_insights
+
+
+def assess_data_quality(df: pd.DataFrame) -> Dict[str, Any]:
+    """
+    Assesses data quality and identifies potential issues.
+    """
+    quality_metrics = {
+        "overall_quality_score": 0,
+        "quality_issues": [],
+        "data_completeness": 0,
+        "data_consistency": {}
+    }
+    
+    # Completeness assessment
+    completeness = (1 - df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100
+    quality_metrics["data_completeness"] = completeness
+    
+    # Identify quality issues
+    if completeness < 95:
+        quality_metrics["quality_issues"].append("Missing data detected")
+    
+    # Check for duplicates
+    duplicate_rows = df.duplicated().sum()
+    if duplicate_rows > 0:
+        quality_metrics["quality_issues"].append(f"{duplicate_rows} duplicate rows found")
+    
+    # Check for inconsistent data types
+    for col in df.columns:
+        if df[col].dtype == 'object':
+            if df[col].str.isnumeric().any() and not df[col].str.isnumeric().all():
+                quality_metrics["quality_issues"].append(f"Inconsistent data types in {col}")
+    
+    # Calculate overall quality score
+    base_score = 100
+    base_score -= (100 - completeness) * 0.5  # Penalize missing data
+    base_score -= len(quality_metrics["quality_issues"]) * 5  # Penalize each quality issue
+    quality_metrics["overall_quality_score"] = max(0, base_score)
+    
+    return quality_metrics
+
+
+def infer_business_context(df: pd.DataFrame, domain_analysis: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Infers business context and potential use cases based on the data characteristics.
+    """
+    domain = domain_analysis["primary_domain"]
+    
+    context_mapping = {
+        "financial": {
+            "key_metrics": ["Revenue", "Profit", "Cost", "ROI"],
+            "typical_analyses": ["Trend analysis", "Profitability analysis", "Budget vs actual"],
+            "stakeholders": ["CFO", "Finance team", "Executive leadership"]
+        },
+        "survey": {
+            "key_metrics": ["Satisfaction scores", "Response rates", "Sentiment"],
+            "typical_analyses": ["Satisfaction analysis", "Demographic breakdown", "Correlation analysis"],
+            "stakeholders": ["Marketing team", "Product managers", "Customer success"]
+        },
+        "scientific": {
+            "key_metrics": ["Statistical significance", "Effect size", "Confidence intervals"],
+            "typical_analyses": ["Hypothesis testing", "Regression analysis", "Experimental validation"],
+            "stakeholders": ["Researchers", "Scientists", "Academic community"]
+        },
+        "marketing": {
+            "key_metrics": ["Conversion rates", "Customer acquisition cost", "Campaign ROI"],
+            "typical_analyses": ["Campaign performance", "Customer segmentation", "Attribution analysis"],
+            "stakeholders": ["Marketing team", "CMO", "Sales team"]
+        }
+    }
+    
+    return context_mapping.get(domain, {
+        "key_metrics": ["Performance indicators", "Trends", "Patterns"],
+        "typical_analyses": ["Descriptive analysis", "Trend identification", "Pattern recognition"],
+        "stakeholders": ["Business stakeholders", "Decision makers"]
+    })
+
+
+def generate_intelligent_report(llm, autonomous_context: Dict[str, Any]) -> str:
+    """
+    Generates an intelligent, domain-appropriate report with organic storytelling.
+    """
+    # Create truly autonomous prompt that lets AI decide everything
+    enhanced_prompt = f"""
+You are a world-class data analyst who has just been handed this dataset to analyze. Look at the data characteristics and tell me the most compelling story you can find.
+
+**DATASET CONTEXT:**
+{json.dumps(autonomous_context, indent=2)}
+
+**YOUR MISSION:**
+Analyze this data like you would if a CEO walked into your office and said "I need to understand what this data is telling us." Write a report that would make them say "This is exactly what I needed to know."
+
+**GUIDELINES:**
+- Don't follow a rigid structure - let the data guide your narrative
+- Choose your own headings and sections based on what the data reveals
+- Write like you're presenting findings to someone who needs to make important decisions
+- Include specific numbers and insights that matter
+- Insert chart recommendations like: `<generate_chart: "chart_type | description">`
+- Valid chart types: bar, pie, line, scatter, hist, box, heatmap
+- Only recommend charts that truly support your narrative
+
+**FORGET TEMPLATES - TELL THE STORY:**
+What's the most interesting, important, or surprising thing this data reveals? Start there and build your entire report around that central insight. Make it compelling, make it actionable, make it memorable.
+
+Be the data analyst who gets promoted because they don't just present data - they reveal insights that drive business decisions.
+"""
+
+
+# Removed - no longer needed since we're letting AI decide everything organically
+
+
+def generate_autonomous_charts(llm, df: pd.DataFrame, report_md: str, uid: str, project_id: str, bucket) -> Dict[str, str]:
+    """
+    Generates charts autonomously based on the report content and data characteristics.
+    """
+    # Extract chart descriptions from the enhanced report
+    chart_descs = extract_chart_tags(report_md)[:MAX_CHARTS]
+    chart_urls = {}
+    
+    if not chart_descs:
+        # If no charts specified, generate intelligent defaults
+        chart_descs = generate_intelligent_chart_suggestions(df, llm)
+    
+    chart_generator = ChartGenerator(llm, df)
+    
+    for desc in chart_descs:
+        try:
+            # Create a safe key for Firebase
+            safe_desc = sanitize_for_firebase_key(desc)
+            
+            # Replace chart tags in markdown
+            report_md = report_md.replace(f'<generate_chart: "{desc}">', f'<generate_chart: "{safe_desc}">')
+            report_md = report_md.replace(f'<generate_chart: {desc}>', f'<generate_chart: "{safe_desc}">')
+            
+            # Generate chart
+            with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file:
+                img_path = Path(temp_file.name)
+                try:
+                    chart_spec = chart_generator.generate_chart_spec(desc)
+                    if execute_chart_spec(chart_spec, df, img_path):
+                        blob_name = f"sozo_projects/{uid}/{project_id}/charts/{uuid.uuid4().hex}.png"
+                        blob = bucket.blob(blob_name)
+                        blob.upload_from_filename(str(img_path))
+                        
+                        chart_urls[safe_desc] = blob.public_url
+                        logging.info(f"Generated autonomous chart: {safe_desc}")
+                finally:
+                    if os.path.exists(img_path):
+                        os.unlink(img_path)
+                        
+        except Exception as e:
+            logging.error(f"Failed to generate chart '{desc}': {str(e)}")
+            continue
+    
+    return chart_urls
+
+
+def generate_intelligent_chart_suggestions(df: pd.DataFrame, llm) -> List[str]:
+    """
+    Generates intelligent chart suggestions based on data characteristics.
+    """
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    categorical_cols = df.select_dtypes(include=['object']).columns
+    
+    suggestions = []
+    
+    # Time series chart if temporal data exists
+    if detect_time_series(df):
+        suggestions.append("line | Time series trend analysis | Show temporal patterns")
+    
+    # Distribution chart for numeric data
+    if len(numeric_cols) > 0:
+        main_numeric = numeric_cols[0]
+        suggestions.append(f"hist | Distribution of {main_numeric} | Understand data distribution")
+    
+    # Correlation analysis if multiple numeric columns
+    if len(numeric_cols) > 1:
+        suggestions.append("scatter | Correlation analysis | Identify relationships between variables")
+    
+    # Categorical breakdown
+    if len(categorical_cols) > 0:
+        main_categorical = categorical_cols[0]
+        suggestions.append(f"bar | {main_categorical} breakdown | Show categorical distribution")
+    
+    return suggestions[:MAX_CHARTS]
+
+
+# Helper functions (preserve existing functionality)
+def detect_time_series(df: pd.DataFrame) -> bool:
+    """Detect if dataset contains time series data."""
+    for col in df.columns:
+        if 'date' in col.lower() or 'time' in col.lower():
+            return True
+        try:
+            pd.to_datetime(df[col])
+            return True
+        except:
+            continue
+    return False
+
+
+def detect_transactional_data(df: pd.DataFrame) -> bool:
+    """Detect if dataset contains transactional data."""
+    transaction_indicators = ['transaction', 'payment', 'order', 'invoice', 'amount', 'quantity']
+    columns_lower = [col.lower() for col in df.columns]
+    return any(indicator in col for col in columns_lower for indicator in transaction_indicators)
+
+
+def detect_experimental_data(df: pd.DataFrame) -> bool:
+    """Detect if dataset contains experimental data."""
+    experimental_indicators = ['test', 'experiment', 'trial', 'group', 'treatment', 'control']
+    columns_lower = [col.lower() for col in df.columns]
+    return any(indicator in col for col in columns_lower for indicator in experimental_indicators)
+
+
+def detect_temporal_frequency(date_series: pd.Series) -> str:
+    """Detect the frequency of temporal data."""
+    if len(date_series) < 2:
+        return "insufficient_data"
+    
+    # Calculate time differences
+    time_diffs = date_series.sort_values().diff().dropna()
+    median_diff = time_diffs.median()
+    
+    if median_diff <= pd.Timedelta(days=1):
+        return "daily"
+    elif median_diff <= pd.Timedelta(days=7):
+        return "weekly"
+    elif median_diff <= pd.Timedelta(days=31):
+        return "monthly"
+    else:
+        return "irregular"
+
+
+def determine_analysis_complexity(df: pd.DataFrame, domain_analysis: Dict[str, Any]) -> str:
+    """Determine the complexity level of analysis required."""
+    complexity_factors = 0
+    
+    # Data size factor
+    if len(df) > 10000:
+        complexity_factors += 1
+    if len(df.columns) > 20:
+        complexity_factors += 1
+    
+    # Data type diversity
+    if len(df.select_dtypes(include=[np.number]).columns) > 5:
+        complexity_factors += 1
+    if len(df.select_dtypes(include=['object']).columns) > 5:
+        complexity_factors += 1
+    
+    # Domain complexity
+    if domain_analysis["primary_domain"] in ["scientific", "financial"]:
+        complexity_factors += 1
+    
+    if complexity_factors >= 3:
+        return "high"
+    elif complexity_factors >= 2:
+        return "medium"
+    else:
+        return "low"
+
+
+def generate_original_report(df: pd.DataFrame, llm, ctx: str, uid: str, project_id: str, bucket) -> Dict[str, str]:
+    """
+    Fallback to original report generation logic if enhanced version fails.
+    """
+    logging.info("Using fallback report generation")
+    
+    # Original logic preserved
     ctx_dict = {"shape": df.shape, "columns": list(df.columns), "user_ctx": ctx}
     enhanced_ctx = enhance_data_context(df, ctx_dict)
+    
     report_prompt = f"""
     You are a senior data analyst and business intelligence expert. Analyze the provided dataset and write a comprehensive executive-level Markdown report.
     **Dataset Analysis Context:** {json.dumps(enhanced_ctx, indent=2)}
@@ -284,37 +812,62 @@ def generate_report_draft(buf, name: str, ctx: str, uid: str, project_id: str, b
        Valid chart types: bar, pie, line, scatter, hist.
     Generate insights that would be valuable to C-level executives.
     """
+    
     md = llm.invoke(report_prompt).content
     chart_descs = extract_chart_tags(md)[:MAX_CHARTS]
     chart_urls = {}
     chart_generator = ChartGenerator(llm, df)
 
     for desc in chart_descs:
-        # Create a safe key for Firebase
         safe_desc = sanitize_for_firebase_key(desc)
-
-        # Replace the original description in the markdown with the safe one
         md = md.replace(f'<generate_chart: "{desc}">', f'<generate_chart: "{safe_desc}">')
-        md = md.replace(f'<generate_chart: {desc}>', f'<generate_chart: "{safe_desc}">') # Handle no quotes case
+        md = md.replace(f'<generate_chart: {desc}>', f'<generate_chart: "{safe_desc}">')
 
         with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file:
             img_path = Path(temp_file.name)
             try:
-                chart_spec = chart_generator.generate_chart_spec(desc) # Still generate spec from original desc
+                chart_spec = chart_generator.generate_chart_spec(desc)
                 if execute_chart_spec(chart_spec, df, img_path):
                     blob_name = f"sozo_projects/{uid}/{project_id}/charts/{uuid.uuid4().hex}.png"
                     blob = bucket.blob(blob_name)
                     blob.upload_from_filename(str(img_path))
-                    
-                    # Use the safe key in the dictionary
                     chart_urls[safe_desc] = blob.public_url
-                    logging.info(f"Uploaded chart '{desc}' to {blob.public_url} with safe key '{safe_desc}'")
             finally:
                 if os.path.exists(img_path):
                     os.unlink(img_path)
 
     return {"raw_md": md, "chartUrls": chart_urls}
 
+
+def generate_fallback_report(autonomous_context: Dict[str, Any]) -> str:
+    """
+    Generates a basic fallback report when enhanced generation fails.
+    """
+    basic_info = autonomous_context["basic_info"]
+    domain = autonomous_context["domain"]["primary_domain"]
+    
+    return f"""
+# What This Data Reveals
+
+Looking at this {domain} dataset with {basic_info['shape'][0]} records, there are several key insights worth highlighting.
+
+## The Numbers Tell a Story
+
+This dataset contains {basic_info['shape'][1]} different variables, suggesting a comprehensive view of the underlying processes or behaviors being measured.
+
+<generate_chart: "bar | Data overview showing key metrics">
+
+## What You Should Know
+
+The data structure and patterns suggest this is worth deeper investigation. The variety of data types and relationships indicate multiple analytical opportunities.
+
+## Next Steps
+
+Based on this initial analysis, I recommend diving deeper into the specific patterns and relationships within the data to unlock more actionable insights.
+
+*Note: This is a simplified analysis. Enhanced storytelling temporarily unavailable.*
+"""
+
 def generate_single_chart(df: pd.DataFrame, description: str, uid: str, project_id: str, bucket):
     logging.info(f"Generating single chart '{description}' for project {project_id}")
     llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", google_api_key=API_KEY, temperature=0.1)