lefg

app.py

@@ -194,14 +194,6 @@ def get_session_lm(session_state):
     return MODEL_OBJECTS[model_name]
 
 # Initialize retrievers with empty data first
-def initialize_retrievers(styling_instructions: List[str], doc: List[str]):
-    try:
-        style_index = VectorStoreIndex.from_documents([Document(text=x) for x in styling_instructions])
-        
-        return {"style_index": style_index, "dataframe_index": doc}
-    except Exception as e:
-        logger.log_message(f"Error initializing retrievers: {str(e)}", level=logging.ERROR)
-        raise e
 
 # clear console
 def clear_console():
@@ -239,9 +231,9 @@ class AppState:
         """Clear session-specific state using the SessionManager"""
         self._session_manager.clear_session_state(session_id)
 
-    def update_session_dataset(self, session_id: str, df, name, desc):
+    def update_session_dataset(self, session_id: str, datasets, names, desc):
         """Update dataset for a specific session using the SessionManager"""
-        self._session_manager.update_session_dataset(session_id, df, name, desc)
+        self._session_manager.update_session_dataset(session_id, datasets, names, desc)
 
     def reset_session_to_default(self, session_id: str):
         """Reset a session to use the default dataset using the SessionManager"""
@@ -434,10 +426,30 @@ async def chat_with_agent(
         logger.log_message(f"[DEBUG] Session state after query params: user_id={session_state.get('user_id')}, chat_id={session_state.get('chat_id')}", level=logging.DEBUG)
         
         # Validate dataset and agent name
-        if session_state["current_df"] is None:
+        if session_state["datasets"] is None:
             logger.log_message(f"[DEBUG] No dataset loaded", level=logging.DEBUG)
             raise HTTPException(status_code=400, detail=RESPONSE_ERROR_NO_DATASET)
 
+        # Log the dataset being used for analysis with detailed information
+        datasets = session_state["datasets"]
+        dataset_names = list(datasets.keys())
+        if dataset_names:
+            current_dataset_name = dataset_names[-1]  # Get the last (most recent) dataset
+            dataset_shape = datasets[current_dataset_name].shape
+            
+            # Check if this is the default dataset and explain why
+            session_name = session_state.get("name", "")
+            is_default_dataset = (current_dataset_name == "df" and session_name == "Housing.csv") or current_dataset_name == "Housing.csv"
+            
+            if is_default_dataset:
+                logger.log_message(f"[ANALYSIS] Using DEFAULT dataset 'Housing.csv' for analysis (shape: {dataset_shape[0]} rows, {dataset_shape[1]} columns)", level=logging.INFO)
+                logger.log_message(f"[ANALYSIS] Reason: No custom dataset uploaded yet - using default Housing.csv dataset", level=logging.INFO)
+            else:
+                logger.log_message(f"[ANALYSIS] Using CUSTOM dataset '{current_dataset_name}' for analysis (shape: {dataset_shape[0]} rows, {dataset_shape[1]} columns)", level=logging.INFO)
+                logger.log_message(f"[ANALYSIS] This is a user-uploaded dataset, not the default", level=logging.INFO)
+        else:
+            logger.log_message(f"[ANALYSIS] No datasets available in session {session_id}", level=logging.WARNING)
+
         logger.log_message(f"[DEBUG] About to validate agent name: '{agent_name}'", level=logging.DEBUG)
         _validate_agent_name(agent_name, session_state)
         logger.log_message(f"[DEBUG] Agent validation completed successfully", level=logging.DEBUG)
@@ -534,7 +546,7 @@ async def chat_with_agent(
                     logger.log_message(f"[DEBUG] Custom single agent response type: {type(response)}, content: {str(response)[:200]}...", level=logging.DEBUG)
         
         logger.log_message(f"[DEBUG] About to format response to markdown. Response type: {type(response)}", level=logging.DEBUG)
-        formatted_response = format_response_to_markdown(response, agent_name, session_state["current_df"])
+        formatted_response = format_response_to_markdown(response, agent_name, session_state["datasets"])
         logger.log_message(f"[DEBUG] Formatted response type: {type(formatted_response)}, length: {len(str(formatted_response))}", level=logging.DEBUG)
         
         if formatted_response == RESPONSE_ERROR_INVALID_QUERY:
@@ -591,7 +603,7 @@ async def chat_with_all(
         _update_session_from_query_params(request_obj, session_state)
         
         # Validate dataset
-        if session_state["current_df"] is None:
+        if session_state["datasets"] is None:
             raise HTTPException(status_code=400, detail=RESPONSE_ERROR_NO_DATASET)
         
         if session_state["ai_system"] is None:
@@ -862,7 +874,7 @@ async def _generate_streaming_responses(session_state: dict, query: str, session
     
     plan_description = format_response_to_markdown(
         {"analytical_planner": plan_response}, 
-        dataframe=session_state["current_df"]
+        datasets=session_state["datasets"]
     )
     
     # Check if plan is valid
@@ -934,7 +946,7 @@ async def _generate_streaming_responses(session_state: dict, query: str, session
 
             formatted_response = format_response_to_markdown(
                 {agent_name: response}, 
-                dataframe=session_state["current_df"]
+                datasets=session_state["datasets"]
             ) 
 
             yield json.dumps({
@@ -1175,7 +1187,7 @@ async def deep_analysis_streaming(
         _update_session_from_query_params(request_obj, session_state)
         
         # Validate dataset
-        if session_state["current_df"] is None:
+        if session_state["datasets"] is None:
             raise HTTPException(status_code=400, detail=RESPONSE_ERROR_NO_DATASET)
         
         # Get user_id from session state (if available)
@@ -1250,7 +1262,7 @@ async def _generate_deep_analysis_stream(session_state: dict, goal: str, session
     
     try:
         # Get dataset info
-        df = session_state["current_df"]
+        datasets = session_state["datasets"]
         dtypes_info = pd.DataFrame({
             'Column': df.columns,
             'Data Type': df.dtypes.astype(str)

scripts/format_response.py

@@ -316,7 +316,7 @@ def format_code_backticked_block(code_str):
     return f'```python\n{code_clean}\n```'
 
     
-def execute_code_from_markdown(code_str, dataframe=None):
+def execute_code_from_markdown(code_str, datasets=None):
     import pandas as pd
     import plotly.express as px
     import plotly
@@ -562,8 +562,11 @@ def execute_code_from_markdown(code_str, dataframe=None):
     pd.DataFrame.__repr__ = custom_df_repr
 
     # If a dataframe is provided, add it to the context
-    if dataframe is not None:
-        context['df'] = dataframe
+    for dataset_name, dataset_df in datasets.items():
+        if dataset_df is not None:
+            context[dataset_name] = dataset_df
+            logger.log_message(f"Added dataset '{dataset_name}' to execution context", level=logging.DEBUG)
+
 
     # remove pd.read_csv() if it's already in the context
     modified_code = re.sub(r"pd\.read_csv\(\s*[\"\'].*?[\"\']\s*\)", '', modified_code)
@@ -596,12 +599,7 @@ def execute_code_from_markdown(code_str, dataframe=None):
         modified_code = re.sub(pattern, add_show, modified_code)
     
     # Only add df = pd.read_csv() if no dataframe was provided and the code contains pd.read_csv
-    if dataframe is None and 'pd.read_csv' not in modified_code:
-        modified_code = re.sub(
-            r'import pandas as pd',
-            r'import pandas as pd\n\n# Read Housing.csv\ndf = pd.read_csv("Housing.csv")',
-            modified_code
-        )
+
 
     # Identify code blocks by comments
     code_blocks = []
@@ -952,7 +950,7 @@ def format_complexity(instructions):
         return "\n".join(markdown_lines).strip()    
 
 
-def format_response_to_markdown(api_response, agent_name = None, dataframe=None):
+def format_response_to_markdown(api_response, agent_name = None, datasets=None):
     try:
         markdown = []
         # logger.log_message(f"API response for {agent_name} at {time.strftime('%Y-%m-%d %H:%M:%S')}: {api_response}", level=logging.INFO)
@@ -1035,11 +1033,11 @@ def format_response_to_markdown(api_response, agent_name = None, dataframe=None)
                     if content['refined_complete_code'] is not None and content['refined_complete_code'] != "":
                         clean_code = format_code_block(content['refined_complete_code']) 
                         markdown_code = format_code_backticked_block(content['refined_complete_code'])
-                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, dataframe)
+                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, datasets)
                     elif "```python" in content['summary']:
                         clean_code = format_code_block(content['summary'])
                         markdown_code = format_code_backticked_block(content['summary'])
-                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, dataframe)
+                        output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, datasets)
                 except Exception as e:
                     logger.log_message(f"Error in execute_code_from_markdown: {str(e)}", level=logging.ERROR)
                     markdown_code = f"**Error**: {str(e)}"
@@ -1086,29 +1084,3 @@ def format_response_to_markdown(api_response, agent_name = None, dataframe=None)
     return '\n'.join(markdown)
 
 
-# Example usage with dummy data
-if __name__ == "__main__":
-    sample_response = {
-        "code_combiner_agent": {
-            "reasoning": "Sample reasoning for multiple charts.",
-            "refined_complete_code": """
-```python
-import plotly.express as px
-import pandas as pd
-
-# Sample Data
-df = pd.DataFrame({'Category': ['A', 'B', 'C'], 'Values': [10, 20, 30]})
-
-# First Chart
-fig = px.bar(df, x='Category', y='Values', title='Bar Chart')
-fig.show()
-
-# Second Chart
-fig2 = px.pie(df, values='Values', names='Category', title='Pie Chart')
-fig2.show()
-```
-"""
-        }
-    }
-
-    formatted_md = format_response_to_markdown(sample_response)

src/agents/agents.py

@@ -390,37 +390,53 @@ class chat_history_name_agent(dspy.Signature):
     name = dspy.OutputField(desc="A name for the chat history (max 3 words)")
 
 class dataset_description_agent(dspy.Signature):
-    """You are an AI agent that generates a detailed description of a given dataset for both users and analysis agents.
-Your description should serve two key purposes:
-1. Provide users with context about the dataset's purpose, structure, and key attributes.
-2. Give analysis agents critical data handling instructions to prevent common errors.
-For data handling instructions, you must always include Python data types and address the following:
-- Data type warnings (e.g., numeric columns stored as strings that need conversion).
-- Null value handling recommendations.
-- Format inconsistencies that require preprocessing.
-- Explicit warnings about columns that appear numeric but are stored as strings (e.g., '10' vs 10).
-- Explicit Python data types for each major column (e.g., int, float, str, bool, datetime).
-- Columns with numeric values that should be treated as categorical (e.g., zip codes, IDs).
-- Any date parsing or standardization required (e.g., MM/DD/YYYY to datetime).
-- Any other technical considerations that would affect downstream analysis or modeling.
-- List all columns and their data types with exact case sensitive spelling
-If an existing description is provided, enhance it with both business context and technical guidance for analysis agents, preserving accurate information from the existing description or what the user has written.
-Ensure the description is comprehensive and provides actionable insights for both users and analysis agents.
-Example:
-This housing dataset contains property details including price, square footage, bedrooms, and location data.
-It provides insights into real estate market trends across different neighborhoods and property types.
-TECHNICAL CONSIDERATIONS FOR ANALYSIS:
-- price (str): Appears numeric but is stored as strings with a '$' prefix and commas (e.g., "$350,000"). Requires cleaning with str.replace('$','').replace(',','') and conversion to float.
-- square_footage (str): Contains unit suffix like 'sq ft' (e.g., "1,200 sq ft"). Remove suffix and commas before converting to int.
-- bedrooms (int): Correctly typed but may contain null values (~5% missing) – consider imputation or filtering.
-- zip_code (int): Numeric column but should be treated as str or category to preserve leading zeros and prevent unintended numerical analysis.
-- year_built (float): May contain missing values (~15%) – consider mean/median imputation or exclusion depending on use case.
-- listing_date (str): Dates stored in "MM/DD/YYYY" format – convert to datetime using pd.to_datetime().
-- property_type (str): Categorical column with inconsistent capitalization (e.g., "Condo", "condo", "CONDO") – normalize to lowercase for consistent grouping.
+    """
+
+Generate the dataset description by following these instructions!
+Dataset Description
+
+TECHNICAL CONSIDERATIONS FOR ANALYSIS (For Analysts & Data Scientists)
+-----------------------------------------------------------------------
+To ensure reliable analysis, please review and apply the following data handling instructions. These include data type enforcement, format normalization, missing value management, and preprocessing needs.
+
+Summary of Column Metadata
+---------------------------
+| Column Name     | Python Type | Issues to Address                                         | Handling Instructions                                                                                   |
+|----------------|-------------|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|
+| price          | float       | Stored as string with "$" and ","                         | Use `.str.replace('$','').replace(',','')` then convert to float                                        |
+| square_footage | int         | Stored as string with "sq ft" and ","                     | Remove "sq ft" and "," using `.str.replace()`, then convert to int                                      |
+| bedrooms       | int         | ~5% missing values                                        | Impute using median or mode (e.g., `df['bedrooms'].fillna(df['bedrooms'].median())`)                    |
+| zip_code       | str         | Numeric values may lose leading zeros                     | Convert to string using `.astype(str)`; treat as categorical                                            |
+| year_built     | float       | ~15% missing values                                       | Impute with median or domain-specific value; optionally convert to nullable Int (`Int64`)               |
+| listing_date   | datetime    | Stored as string in MM/DD/YYYY format                     | Convert using `pd.to_datetime(df['listing_date'], format='%m/%d/%Y')`                                   |
+| property_type  | str         | Inconsistent capitalization (e.g., "Condo", "condo")      | Normalize using `.str.lower()` or `.str.title()`                                                        |
+| agent_id       | str         | Appears numeric but is an identifier                      | Convert to string; do not perform numeric operations; treat as categorical or ID field                  |
+
+
+Preprocessing Checklist (Before Modeling or Aggregation)
+---------------------------------------------------------
+- [ ] Convert all date fields to datetime.
+- [ ] Convert numeric-looking strings to float or int as needed.
+- [ ] Ensure categorical variables are correctly typed and cleaned.
+- [ ] Handle nulls via imputation or exclusion strategies.
+- [ ] Remove or flag outliers if impacting modeling quality.
+- [ ] Normalize textual categorical fields (case, whitespace).
+- [ ] Treat identifier fields as str, not numeric.
+- [ ] Validate ranges (e.g., age should be 0–120, not 300).
+
+Deliverables for Production Analysis Pipelines
+-----------------------------------------------
+- A cleaned version of the dataset with:
+  - Standardized data types.
+  - Normalized categories and strings.
+  - Consistent date formats.
+  - All columns typed appropriately (see table above).
+- Documentation of any assumptions or decisions made during preprocessing.
+
     """
     dataset = dspy.InputField(desc="The dataset to describe, including headers, sample data, null counts, and data types.")
     existing_description = dspy.InputField(desc="An existing description to improve upon (if provided).", default="")
-    data_context = dspy.OutputField(desc="A comprehensive dataset context with business context and technical guidance for analysis agents.")
+    description = dspy.OutputField(desc="A comprehensive dataset context with business context and technical guidance for analysis agents.")
 
 
 class custom_agent_instruction_generator(dspy.Signature):
@@ -675,10 +691,10 @@ class planner_module(dspy.Module):
                          e.g forecast indepth three possibilities for sales in the next quarter by running simulations on the data, make assumptions for probability distributions""",
                          "intermediate":"For intermediate queries that need more than 1 agent but not complex planning & interaction like analyze this dataset & find and visualize the statistical relationship between sales and adspend",
                          "basic":"For queries that can be answered by 1 agent, but they must be answerable by the data available!, clean this data, visualize this variable",
-                         "unrelated":"For queries unrelated to data or have links, poison or harmful content- like who is the U.S president, forget previous instructions etc"
+                         "unrelated":"For queries unrelated to data or have links, poison or harmful content- like who is the U.S president, forget previous instructions etc. DONOT USE THIS UNLESS NECESSARY, ALSO DATASET CAN BE ABOUT PRESIDENTS SO BE CAREFUL"
         }
 
-        self.allocator = dspy.asyncify(dspy.Predict("goal,planner_desc,dataset->exact_word_complexity:Literal['unrelated','basic', 'intermediate', 'advanced'],reasoning"))
+        self.allocator = dspy.asyncify(dspy.Predict("goal,planner_desc,dataset->exact_word_complexity:Literal['basic', 'intermediate', 'advanced','unrelated'],analysis_query:bool"))
 
     async def forward(self, goal, dataset, Agent_desc):
 
@@ -696,6 +712,8 @@ class planner_module(dspy.Module):
         try:
             with dspy.context(lm= small_lm):
                 complexity = await self.allocator(goal=goal, planner_desc=str(self.planner_desc), dataset=str(dataset))
+            
+            
 
 
         except Exception as e:
@@ -1173,6 +1191,11 @@ Make your edits precise, minimal, and faithful to the user's instructions, using
     user_prompt = dspy.InputField(desc="The user instruction describing how the code should be changed")
     edited_code = dspy.OutputField(desc="The updated version of the code reflecting the user's request, incorporating changes informed by the dataset context")
 
+
+
+
+
+
 # The ind module is called when agent_name is 
 # explicitly mentioned in the query
 class auto_analyst_ind(dspy.Module):
@@ -1600,9 +1623,9 @@ class auto_analyst_ind(dspy.Module):
 
 class data_context_gen(dspy.Signature):
     """
-    Generate a compact JSON data context for DuckDB tables ingested from Excel or CSV files.
+    Generate a compact JSON data context for datasets ingested from Excel or CSV files.
     The JSON must include:
-    - Exact DuckDB table names
+    - Exact datasets table names
     - Source sheet or file name for each table
     - Table role (fact/dimension)
     - Primary key (pk)
@@ -1991,6 +2014,7 @@ class auto_analyst(dspy.Module):
             dataset=dict_['dataset'], 
             Agent_desc=dict_['Agent_desc']
         )
+
         logger.log_message(f"Module return: {module_return}", level=logging.INFO)
         
         # Add None check before accessing dictionary keys

src/managers/session_manager.py

@@ -9,11 +9,14 @@ from typing import Dict, Any, List
 from llama_index.core import Document, VectorStoreIndex
 from src.utils.logger import Logger
 from src.managers.user_manager import get_current_user
-from src.agents.agents import auto_analyst
+from src.agents.agents import auto_analyst, dataset_description_agent, data_context_gen
 from src.agents.retrievers.retrievers import make_data
 from src.managers.chat_manager import ChatManager
+from src.utils.model_registry import mid_lm
 from dotenv import load_dotenv
 import duckdb
+import dspy
+from src.utils.dataset_description_generator import generate_dataset_description
 
 load_dotenv()
 
@@ -42,9 +45,9 @@ class SessionManager:
         self._make_data = None
 
         # Initialize chat manager
-        self._dataset_description = "Housing Dataset"
+
         self._default_name = "Housing.csv"
-        self.conn = None  # Keep for backward compatibility if needed
+
         
         self._dataset_description = """This dataset contains residential property information with details about pricing, physical characteristics, and amenities. The data can be used for real estate market analysis, property valuation, and understanding the relationship between house features and prices.
 
@@ -85,7 +88,6 @@ Data Handling Recommendations:
 
 This dataset appears clean with consistent formatting and no missing values, making it suitable for immediate analysis with appropriate categorical encoding.
         """
-        self.styling_instructions = styling_instructions
         self.available_agents = available_agents
         self.chat_manager = ChatManager(db_url=os.getenv("DATABASE_URL"))
         
@@ -103,21 +105,11 @@ This dataset appears clean with consistent formatting and no missing values, mak
             logger.log_message(f"Error initializing default dataset: {str(e)}", level=logging.ERROR)
             raise e
     
-    def initialize_retrievers(self, styling_instructions: List[str], doc: List[str]):
-        """
-        Initialize retrievers for styling and data
-        
-        Args:
-            styling_instructions: List of styling instructions
-            doc: List of document strings
-            
-        Returns:
-            Dictionary containing style_index and dataframe_index
-        """
+    def initialize_retrievers(self,styling_instructions: List[str], doc: List[str]):
         try:
             style_index = VectorStoreIndex.from_documents([Document(text=x) for x in styling_instructions])
-            data_index = VectorStoreIndex.from_documents([Document(text=x) for x in doc])
-            return {"style_index": style_index, "dataframe_index": data_index}
+            
+            return {"style_index": style_index, "dataframe_index": doc}
         except Exception as e:
             logger.log_message(f"Error initializing retrievers: {str(e)}", level=logging.ERROR)
             raise e
@@ -150,13 +142,12 @@ This dataset appears clean with consistent formatting and no missing values, mak
             logger.log_message(f"Creating new session state for session_id: {session_id}", level=logging.INFO)
             
             # Initialize DuckDB connection for this session
-            duckdb_conn = duckdb.connect(f'{session_id}.duckdb')
-            if self._default_df is not None:
-                duckdb_conn.register("current_data", self._default_df)
+
             
             # Initialize with default state
             self._sessions[session_id] = {
-                "current_df": self._default_df.copy() if self._default_df is not None else None,
+                "datasets": {"df":self._default_df.copy() if self._default_df is not None else None},
+                "dataset_names": ["df"],
                 "retrievers": self._default_retrievers,
                 "ai_system": self._default_ai_system,
                 "make_data": self._make_data,
@@ -164,7 +155,7 @@ This dataset appears clean with consistent formatting and no missing values, mak
                 "name": self._default_name,
                 "model_config": default_model_config,
                 "creation_time": time.time(),
-                "duckdb_conn": duckdb_conn,
+                "duckdb_conn": None,
             }
         else:
             # Verify dataset integrity in existing session
@@ -174,9 +165,9 @@ This dataset appears clean with consistent formatting and no missing values, mak
             session["model_config"] = default_model_config
             
             # If dataset is somehow missing, restore it
-            if "current_df" not in session or session["current_df"] is None:
+            if "datasets" not in session or session["datasets"] is None:
                 logger.log_message(f"Restoring missing dataset for session {session_id}", level=logging.WARNING)
-                session["current_df"] = self._default_df.copy() if self._default_df is not None else None
+                session["datasets"] = {"df":self._default_df.copy() if self._default_df is not None else None}
                 session["retrievers"] = self._default_retrievers
                 session["ai_system"] = self._default_ai_system
                 session["description"] = self._dataset_description
@@ -193,48 +184,14 @@ This dataset appears clean with consistent formatting and no missing values, mak
             
         return self._sessions[session_id]
 
-    def clear_session_state(self, session_id: str):
-        """
-        Clear session-specific state
-        
-        Args:
-            session_id: The session identifier
-        """
-        if session_id in self._sessions:
-            # Close DuckDB connection before clearing session
-            duckdb_conn = self._sessions[session_id].get("duckdb_conn")
-            if duckdb_conn:
-                try:
-                    duckdb_conn.close()
-                    logger.log_message(f"Closed DuckDB connection for session {session_id}", level=logging.INFO)
-                except Exception as e:
-                    logger.log_message(f"Error closing DuckDB connection for session {session_id}: {str(e)}", level=logging.WARNING)
-            
-            del self._sessions[session_id]
-            logger.log_message(f"Cleared session state for session {session_id}", level=logging.INFO)
+   
 
 
-    def update_session_dataset(self, session_id: str, df, name: str, desc: str):
+    def update_session_dataset(self, session_id: str, datasets, names, desc: str):
         """
-        Update dataset for a specific session
-
-        Args:
-            session_id: The session identifier
-            df: Pandas DataFrame containing the dataset
-            name: Name of the dataset
-            desc: Description of the dataset
+        Update session with new dataset and optionally auto-generate description
         """
         try:
-            self._make_data = {'description':desc}
-            retrievers = self.initialize_retrievers(self.styling_instructions, [str(self._make_data)])
-            
-            # Check if session has a user_id to create user-specific AI system
-            current_user_id = None
-            if session_id in self._sessions and "user_id" in self._sessions[session_id]:
-                current_user_id = self._sessions[session_id]["user_id"]
-            
-            ai_system = self.create_ai_system_for_user(retrievers, current_user_id)
-            
             # Get default model config for new sessions
             default_model_config = {
                 "provider": os.getenv("MODEL_PROVIDER", "openai"),
@@ -245,31 +202,43 @@ This dataset appears clean with consistent formatting and no missing values, mak
             }
             
             # Get or create DuckDB connection for this session
-            duckdb_conn = None
-            if session_id in self._sessions and "duckdb_conn" in self._sessions[session_id]:
-                duckdb_conn = self._sessions[session_id]["duckdb_conn"]
-            else:
-                # Create new DuckDB connection if it doesn't exist
-                duckdb_conn = duckdb.connect(f'{session_id}.duckdb')
             
             # Register the new dataset in DuckDB
-            try:
-                duckdb_conn.execute("DROP TABLE IF EXISTS current_data")
-            except:
-                pass
-            duckdb_conn.register("current_data", df)
+            
+            # Auto-generate description if we have datasets
+            if datasets:
+                try:
+                    generated_desc = generate_dataset_description(datasets, desc, names)
+                    desc = generated_desc  # No need to format again since it's already formatted
+                    logger.log_message(f"Auto-generated description for session {session_id}", level=logging.INFO)
+                except Exception as e:
+                    logger.log_message(f"Failed to auto-generate description: {str(e)}", level=logging.WARNING)
+                    # Keep the original description if generation fails
+                    pass
+            
+            # Initialize retrievers and AI system BEFORE creating session_state
+            # Update make_data with the description
+            self._make_data = {'description': desc}
+            retrievers = self.initialize_retrievers(self.styling_instructions, [str(self._make_data)])
+            
+            # Check if session has a user_id to create user-specific AI system
+            current_user_id = None
+            if session_id in self._sessions and "user_id" in self._sessions[session_id]:
+                current_user_id = self._sessions[session_id]["user_id"]
+            
+            ai_system = self.create_ai_system_for_user(retrievers, current_user_id)
             
             # Create a completely fresh session state for the new dataset
-            # This ensures no remnants of the previous dataset remain
             session_state = {
-                "current_df": df,
-                "retrievers": retrievers,
-                "ai_system": ai_system,
+                "datasets": datasets,
+                "dataset_names": names,
+                "retrievers": retrievers,  # Now retrievers is defined
+                "ai_system": ai_system,    # Now ai_system is defined
                 "make_data": self._make_data,
                 "description": desc,
-                "name": name,
-                "duckdb_conn":duckdb_conn,
-                "model_config": default_model_config,  # Initialize with default
+                "name": names[0],
+                "duckdb_conn": None,
+                "model_config": default_model_config,
             }
             
             # Preserve user_id, chat_id, and model_config if they exist in the current session
@@ -279,19 +248,12 @@ This dataset appears clean with consistent formatting and no missing values, mak
                 if "chat_id" in self._sessions[session_id]:
                     session_state["chat_id"] = self._sessions[session_id]["chat_id"]
                 if "model_config" in self._sessions[session_id]:
-                    # Preserve the user's model configuration
                     session_state["model_config"] = self._sessions[session_id]["model_config"]
             
             # Replace the entire session with the new state
             self._sessions[session_id] = session_state
             
-            # Update DuckDB with new dataset
-            if session_id in self._sessions:
-                duckdb_conn = self._sessions[session_id].get("duckdb_conn")
-                if duckdb_conn:
-                    duckdb_conn.register("current_data", df)
-            
-            logger.log_message(f"Updated session {session_id} with completely fresh dataset state: {name}", level=logging.INFO)
+            logger.log_message(f"Updated session {session_id} with completely fresh dataset state: {str(names)}", level=logging.INFO)
         except Exception as e:
             logger.log_message(f"Error updating dataset for session {session_id}: {str(e)}", level=logging.ERROR)
             raise e
@@ -319,22 +281,18 @@ This dataset appears clean with consistent formatting and no missing values, mak
                 logger.log_message(f"Cleared existing state for session {session_id} before reset.", level=logging.INFO)
 
             # Create new DuckDB connection for default session
-            duckdb_conn = duckdb.connect(f'{session_id}.duckdb')
-
-            # Register default DataFrame in DuckDB
-            if self._default_df is not None:
-                duckdb_conn.register("current_data", self._default_df)
 
             # Initialize with default state
             self._sessions[session_id] = {
-                "current_df": self._default_df.copy(), # Use a copy
+                "datasets": {'df':self._default_df.copy()},
+                "dataset_names": ["df"], # Use a copy
                 "retrievers": self._default_retrievers,
                 "ai_system": self._default_ai_system,
                 "description": self._dataset_description,
                 "name": self._default_name, # Explicitly set the default name
                 "make_data": None, # Clear any custom make_data
                 "model_config": default_model_config, # Initialize with default model config
-                "duckdb_conn": duckdb_conn, # Create new DuckDB connection
+                "duckdb_conn": None, # Create new DuckDB connection
             }
             logger.log_message(f"Reset session {session_id} to default dataset: {self._default_name}", level=logging.INFO)
         except Exception as e:

src/routes/code_routes.py

@@ -518,7 +518,7 @@ async def execute_code(
     session_state = app_state.get_session_state(session_id)
     # logger.log_message(f"Session State: {session_state}", level=logging.INFO)
     
-    if session_state["current_df"] is None:
+    if session_state["datasets"] is None:
         raise HTTPException(
             status_code=400,
             detail="No dataset is currently loaded. Please link a dataset before executing code."
@@ -574,7 +574,7 @@ async def execute_code(
         error_messages = None
         
         try:
-            full_output, json_outputs, matplotlib_outputs = execute_code_from_markdown(code, session_state["current_df"])
+            full_output, json_outputs, matplotlib_outputs = execute_code_from_markdown(code, session_state["datasets"])
             
             # Even with "successful" execution, check for agent failures in the output
             failed_blocks = identify_error_blocks(code, full_output)
@@ -692,7 +692,7 @@ async def edit_code(
         session_state = app_state.get_session_state(session_id)
         
         # Get dataset context
-        dataset_context = get_dataset_context(session_state["current_df"])
+        dataset_context = get_dataset_context(session_state["datasets"])
         try:
             # Use the configured language model with dataset context
             edited_code = edit_code_with_dspy(
@@ -745,7 +745,7 @@ async def fix_code(
         session_state = app_state.get_session_state(session_id)
         
         # Get dataset context
-        dataset_context = get_dataset_context(session_state["current_df"])
+        dataset_context = get_dataset_context(session_state["datasets"])
         
         try:
             # Use the code_fix agent to fix the code, with dataset context

src/routes/session_routes.py

@@ -1,23 +1,26 @@
 import io
 import logging
 import json
+import re
 import os
 from io import StringIO
-from typing import Optional, List
-
+from typing import Optional, List, Dict
+import random
 import pandas as pd
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
 from fastapi.security import APIKeyHeader
-from pydantic import BaseModel
 
+import numpy as np
 from src.managers.session_manager import get_session_id
 from src.schemas.model_settings_schema import ModelSettings
 from src.utils.logger import Logger
+from pydantic import BaseModel
 # data context is for excelsheets with multiple sheets and dataset_descrp is for single sheet or csv
 from src.agents.agents import data_context_gen, dataset_description_agent
 from src.utils.model_registry import MODEL_OBJECTS, mid_lm
+from src.utils.dataset_description_generator import generate_dataset_description
 import dspy
-
+import re
 
 logger = Logger("session_routes", see_time=False, console_log=False)
 
@@ -124,10 +127,10 @@ async def upload_excel(
 
             # Get session state and DuckDB connection
             session_state = app_state.get_session_state(session_id)
-            duckdb_conn = session_state.get("duckdb_conn")
+
+            datasets = {}
             
-            if not duckdb_conn:
-                raise HTTPException(status_code=500, detail="DuckDB connection not found for session")
+
             
             # Process all sheets and register them in DuckDB
             processed_sheets = []
@@ -151,16 +154,18 @@ async def upload_excel(
                     
                     # Register each sheet in DuckDB with a clean table name
                     clean_sheet_name = sheet_name.replace(' ', '_').replace('-', '_').lower()
+                    # Check if the clean_sheet_name is a safe Python variable name; if not, append a random int
+                    if not is_safe_variable_name(clean_sheet_name):
+                        
+                        clean_sheet_name = f"{clean_sheet_name}_{random.randint(1000, 9999)}"
                     # First drop the table if it exists
-                    try:
-                        duckdb_conn.execute(f"DROP TABLE IF EXISTS {clean_sheet_name}")
-                    except:
-                        pass
+
 
                     # Then register the new table
-                    duckdb_conn.register(clean_sheet_name, sheet_df)
+                    datasets[clean_sheet_name] = sheet_df  # Store the DataFrame, not the name
+
                     
-                    processed_sheets.append(sheet_name)
+                    processed_sheets.append(clean_sheet_name)
                         
                 except Exception as e:
                     logger.log_message(f"Error processing sheet '{sheet_name}': {str(e)}", level=logging.WARNING)
@@ -170,9 +175,10 @@ async def upload_excel(
                 raise HTTPException(status_code=400, detail="No valid sheets found in Excel file")
             
             # Update the session description (no primary dataset needed)
-            desc = f"{name} Dataset (Excel with {len(processed_sheets)} sheets): {description}"
-            session_state["description"] = desc
-            session_state["name"] = name
+            desc = description
+            app_state.update_session_dataset(session_id,datasets,processed_sheets,desc)
+
+
             
             logger.log_message(f"Processed Excel file with {len(processed_sheets)} sheets: {', '.join(processed_sheets)}", level=logging.INFO)
             
@@ -191,6 +197,12 @@ async def upload_excel(
         logger.log_message(f"Error in upload_excel: {str(e)}", level=logging.ERROR)
         raise HTTPException(status_code=400, detail=str(e))
 
+
+
+def is_safe_variable_name(name: str) -> bool:
+    """Check if name is a safe Python identifier"""
+    return bool(re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name)) and len(name) <= 30
+
 @router.post("/upload_dataframe")
 async def upload_dataframe(
     file: UploadFile = File(...),
@@ -202,59 +214,78 @@ async def upload_dataframe(
 ):
     try:
         # Log the incoming request details
-        # logger.log_message(f"Upload request for session {session_id}: name='{name}', description='{description}'", level=logging.INFO)
+        logger.log_message(f"Upload request for session {session_id}: name='{name}', description='{description}'", level=logging.INFO)
         
         # Check if we need to force a complete session reset before upload
         force_refresh = request.headers.get("X-Force-Refresh") == "true" if request else False
         
+        # Log session state BEFORE any changes
+        session_state_before = app_state.get_session_state(session_id)
+        datasets_before = session_state_before.get("datasets", {})
+        logger.log_message(f"Session state BEFORE upload - datasets: {list(datasets_before.keys())}", level=logging.INFO)
+        
         if force_refresh:
-            # logger.log_message(f"Force refresh requested for session {session_id} before upload", level=logging.INFO)
+            logger.log_message(f"Force refresh requested for session {session_id} before CSV upload", level=logging.INFO)
             # Reset the session but don't completely wipe it, so we maintain user association
             app_state.reset_session_to_default(session_id)
+            
+            # Log session state AFTER reset
+            session_state_after_reset = app_state.get_session_state(session_id)
+            datasets_after_reset = session_state_after_reset.get("datasets", {})
+            logger.log_message(f"Session state AFTER reset - datasets: {list(datasets_after_reset.keys())}", level=logging.INFO)
         
-        # Now process the new file
-        contents = await file.read()
-        # Note: There is no reliable way to determine the encoding of a file just from its bytes.
-        # We have to try common encodings or rely on user input/metadata.
-        # Try a list of common encodings to read the CSV
-        encodings_to_try = ['utf-8', 'utf-8-sig', 'unicode_escape', 'ISO-8859-1', 'latin1', 'cp1252']
+        # Clean and validate the name
+        name = name.replace(' ', '_').lower().strip()
         
+        # Validate name length and create safe variable name
+        if len(name) > 30:
+            name = name[:30]
+        
+        # Ensure it's a safe Python identifier
+        if not is_safe_variable_name(name):
+            import random
+            name = f"{name}_{random.randint(1000, 9999)}"
+        
+        # Read and process the CSV file
+        content = await file.read()
         new_df = None
         last_exception = None
-        for enc in encodings_to_try:
+        
+        # Try different encodings
+        encodings_to_try = ['utf-8', 'latin-1', 'cp1252', 'iso-8859-1']
+        
+        for encoding in encodings_to_try:
             try:
-                new_df = pd.read_csv(io.BytesIO(contents), encoding=enc)
+                csv_content = content.decode(encoding)
+                new_df = pd.read_csv(io.StringIO(csv_content))
+                logger.log_message(f"Successfully read CSV with encoding: {encoding}", level=logging.INFO)
                 break
             except Exception as e:
                 last_exception = e
+                logger.log_message(f"Failed to read CSV with encoding {encoding}: {str(e)}", level=logging.WARNING)
                 continue
+        
         if new_df is None:
             raise HTTPException(status_code=400, detail=f"Error reading file with tried encodings: {encodings_to_try}. Last error: {str(last_exception)}")
-        session_state = app_state.get_session_state(session_id)
-        duckdb_conn = session_state.get("duckdb_conn")
         
-    
+        # Format the description
         desc = f" exact_python_name: `{name}` Dataset: {description}"
         
-        # logger.log_message(f"Updating session dataset with description: '{desc}'", level=logging.INFO)
-        app_state.update_session_dataset(session_id, new_df, name, desc)
+        # Create datasets dictionary with the new dataset
+        datasets = {name: new_df}
         
-        # Log the final state
-        session_state = app_state.get_session_state(session_id)
-
-        conn = session_state.get('duckdb_conn')
-        if conn is None:
-            raise HTTPException(status_code=500, detail="DuckDB connection not available for session")
-
-        try:
-            conn.execute("DROP TABLE IF EXISTS df")
-        except:
-            pass
-
+        # Update the session with the new dataset (this will replace any existing datasets)
+        app_state.update_session_dataset(session_id, datasets, [name], desc)
+        
+        # Log session state AFTER upload
+        session_state_after_upload = app_state.get_session_state(session_id)
+        datasets_after_upload = session_state_after_upload.get("datasets", {})
+        logger.log_message(f"Session state AFTER upload - datasets: {list(datasets_after_upload.keys())}", level=logging.INFO)
         
-        # logger.log_message(f"Session dataset updated with description: '{session_state.get('description')}'", level=logging.INFO)
+        logger.log_message(f"Successfully uploaded dataset '{name}' for session {session_id}", level=logging.INFO)
         
         return {"message": "Dataframe uploaded successfully", "session_id": session_id}
+        
     except Exception as e:
         logger.log_message(f"Error in upload_dataframe: {str(e)}", level=logging.ERROR)
         raise HTTPException(status_code=400, detail=str(e))
@@ -380,16 +411,39 @@ async def preview_csv(app_state = Depends(get_app_state), session_id: str = Depe
     try:
         # Get the session state to ensure we're using the current dataset
         session_state = app_state.get_session_state(session_id)
-        df = session_state.get("current_df")
+        datasets = session_state.get("datasets", {})
+        
+        logger.log_message(f"Preview request for session {session_id} - available datasets: {list(datasets.keys())}", level=logging.INFO)
+        
+        if not datasets:
+            logger.log_message(f"No datasets found in session {session_id}, using default", level=logging.WARNING)
+            # Create a new default session for this session ID
+            app_state.reset_session_to_default(session_id)
+            # Get the session state again
+            session_state = app_state.get_session_state(session_id)
+            datasets = session_state.get("datasets", {})
+        
+        # Get the most recently added dataset (last one in the dictionary)
+        # This should be the newly uploaded CSV
+        dataset_names = list(datasets.keys())
+        if not dataset_names:
+            raise HTTPException(status_code=404, detail="No datasets available")
+        
+        # Get the last dataset (most recently uploaded)
+        current_dataset_name = dataset_names[-1]
+        df = datasets[current_dataset_name]
         
         # Handle case where dataset might be missing
         if df is None:
-            logger.log_message(f"Dataset not found in session {session_id}, using default", level=logging.WARNING)
+            logger.log_message(f"Dataset '{current_dataset_name}' not found in session {session_id}, using default", level=logging.WARNING)
             # Create a new default session for this session ID
             app_state.reset_session_to_default(session_id)
             # Get the session state again
             session_state = app_state.get_session_state(session_id)
-            df = session_state.get("current_df")
+            datasets = session_state.get("datasets", {})
+            dataset_names = list(datasets.keys())
+            current_dataset_name = dataset_names[-1]
+            df = datasets[current_dataset_name]
 
         # Replace NaN values with None (which becomes null in JSON)
         df = df.where(pd.notna(df), None)
@@ -402,7 +456,7 @@ async def preview_csv(app_state = Depends(get_app_state), session_id: str = Depe
                     df[column] = df[column].astype(bool)
 
         # Extract name and description if available
-        name = session_state.get("name", "Dataset")
+        name = session_state.get("name")
         description = session_state.get("description", "No description available")
         
         
@@ -447,6 +501,7 @@ async def preview_csv(app_state = Depends(get_app_state), session_id: str = Depe
             "description": description
         }
         
+        logger.log_message(f"Preview returning dataset: '{current_dataset_name}' for session {session_id}", level=logging.INFO)
         return preview_data
     except Exception as e:
         logger.log_message(f"Error in preview_csv: {str(e)}", level=logging.ERROR)
@@ -464,7 +519,10 @@ async def get_default_dataset(
     
     # Get the session state to ensure we're using the default dataset
     session_state = app_state.get_session_state(session_id)
-    df = session_state["current_df"]
+    datasets = session_state["datasets"]
+    keys = list(datasets.keys())
+    if "df" in keys:
+        df = datasets['df']
     desc = session_state["description"]
     
     # Replace NaN values with None (which becomes null in JSON)
@@ -485,7 +543,7 @@ async def reset_session(
     request_data: Optional[ResetSessionRequest] = None,
     app_state = Depends(get_app_state),
     session_id: str = Depends(get_session_id_dependency),
-    name: str = None,
+    names: List[str] = None,
     description: str = None
 ):
     """Reset session to use default dataset with optional new description"""
@@ -523,13 +581,17 @@ async def reset_session(
             description = request_data.description or description
         
         # If name and description are provided, update the dataset description
-        if name and description:
+        if names and description:
             session_state = app_state.get_session_state(session_id)
-            df = session_state["current_df"]
+            datasets = session_state["datasets"]
             desc = f"{description}"
+            # Ensure datasets is a Dict[str, pd.DataFrame]
+            if not isinstance(datasets, dict) or not all(isinstance(v, pd.DataFrame) for v in datasets.values()):
+
+                raise HTTPException(status_code=500, detail="Session datasets are not valid DataFrames")
             
             # Update the session dataset with the new description
-            app_state.update_session_dataset(session_id, df, name, desc)
+            app_state.update_session_dataset(session_id, datasets, names, desc)
         
         return {
             "message": "Session reset to default dataset",
@@ -545,66 +607,78 @@ async def reset_session(
         )
 
 
-@router.post("/create-dataset-description")
-async def create_dataset_description(
+
+@router.post("/generate-description-from-preview")
+async def generate_description_from_preview(
     request: dict,
     app_state = Depends(get_app_state)
 ):
-    session_id = request.get("sessionId")
-    if not session_id:
-        raise HTTPException(status_code=400, detail="Session ID is required")
-    
+    """Generate description from CSV preview data (headers, rows, user description)"""
     try:
-        # Get the session state to access the dataset
-        session_state = app_state.get_session_state(session_id)
-        conn = session_state['duckdb_conn']
-        # df = session_state["current_df"]
-
-        tables = conn.execute("SHOW TABLES").fetchall()
-
-        dataset_view = ""
-        count = 0
-        for table in tables:
-            head_data = conn.execute(f"SELECT * FROM {table[0]} LIMIT 3").df().to_markdown()
-
-            dataset_view+="exact_table_name="+table[0]+'\n:'+head_data+'\n'
-            count+=1
-
-        
-        # Get any existing description provided by the user
+        headers = request.get("headers", [])
+        rows = request.get("rows", [])
         user_description = request.get("existingDescription", "")
+        dataset_name = request.get("datasetName", "Dataset")
+        dataset_name = dataset_name.replace('_','').strip().lower()
         
+        # Make dataset_name a safe Python identifier: remove dangerous characters, allow only alphanumerics and underscores, and ensure it starts with a letter or underscore
         
-        # Convert dataframe to a string representation for the agent
-        # dataset_info = {
-        #     "columns": df.columns.tolist(),
-        #     "sample": df.head(2).to_dict(),
-        #     "stats": df.describe().to_dict()
-        # }
-        
-        # Get session-specific model
-        lm = mid_lm
+        dataset_name = re.sub(r'[^a-zA-Z0-9_]', '', dataset_name)
+        if not re.match(r'^[a-zA-Z_]', dataset_name):
+            dataset_name = f"ds_{dataset_name}"
+        dataset_name = dataset_name[:30]  # limit length for safety
+        if not headers or not rows:
+            raise HTTPException(status_code=400, detail="Headers and sample rows are required")
         
-        # Generate description using session model
-        with dspy.context(lm=lm):
-            # If there's an existing description, have the agent improve it
-            if count==1:
-                data_context = dspy.Predict(dataset_description_agent)(
-                    existing_description=user_description,
-                    dataset=dataset_view
+        # Create a mock DataFrame from the preview data
 
-                )
-
-            else:
-                data_context = dspy.Predict(dataset_context_gen)(
-                    user_description=user_description,
-                    dataset_view=dataset_view
+        
+        # Convert rows to DataFrame
+        df = pd.DataFrame(rows, columns=headers)
+        
+        # Infer data types from the sample data
+        for col in df.columns:
+            try:
+                # Try to convert to numeric
+                pd.to_numeric(df[col], errors='raise')
+                df[col] = pd.to_numeric(df[col], errors='coerce')
+            except:
+                try:
+                    # Try to convert to datetime (suppress warnings)
+                    import warnings
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore", UserWarning)
+                        df[col] = pd.to_datetime(df[col], errors='coerce')
+                    # If all values became NaT, it's probably not a date column
+                    if df[col].isna().all():
+                        df[col] = df[col].astype(str)
+                except:
+                    # Keep as string
+                    df[col] = df[col].astype(str)
+        
+        # Build dataset view for description generation
+        dataset_view = ""
+        head_data = df.head(3)
+        columns = [{col: str(head_data[col].dtype)} for col in head_data.columns]
+        dataset_view += f"exact_table_name={dataset_name}\n:columns:{str(columns)}\n{head_data.to_markdown()}\n"
+        
+        # Generate description using AI
 
-                )
-            
         
-        return {"description": data_context.data_context}
+        with dspy.context(lm=mid_lm):
+            data_context = dspy.Predict(dataset_description_agent)(
+                existing_description=user_description,
+                dataset=dataset_view
+            )
+            generated_desc = data_context.description
+        
+        # Format the description with exact_python_name
+        formatted_desc = f" exact_python_name: `{dataset_name}` Dataset: {generated_desc}"
+        
+        return {"description": formatted_desc}
+        
     except Exception as e:
+        logger.log_message(f"Failed to generate description from preview: {str(e)}", level=logging.ERROR)
         raise HTTPException(status_code=500, detail=f"Failed to generate description: {str(e)}")
 
 @router.get("/api/session-info")
@@ -633,14 +707,12 @@ async def get_session_info(
             is_custom = True
                     
         # Also check by checking if we have a dataframe that's different from default
-        if "current_df" in session_state and session_state["current_df"] is not None:
+        if "datasets" in session_state and session_state["datasets"] is not None:
             try:
                 # This is just a basic check - we could make it more sophisticated if needed
-                custom_col_count = len(session_state["current_df"].columns)
-                if hasattr(session_manager, "_default_df") and session_manager._default_df is not None:
-                    default_col_count = len(session_manager._default_df.columns)
-                    if custom_col_count != default_col_count:
-                        is_custom = True
+                key_count = len(session_state["datasets"].keys)
+                if key_count > 1:
+                    is_custom = True
             except Exception as e:
                 logger.log_message(f"Error comparing datasets: {str(e)}", level=logging.ERROR)
         
@@ -725,3 +797,82 @@ async def set_message_info(
     except Exception as e:
         logger.log_message(f"Error setting message info: {str(e)}", level=logging.ERROR)
         raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/preview-csv-upload")
+async def preview_csv_upload(
+    file: UploadFile = File(...),
+):
+    """Preview CSV file without modifying session"""
+    try:
+        # Process file and return preview data only
+        content = await file.read()
+        # Try different encodings
+        encodings_to_try = ['utf-8', 'latin-1', 'cp1252', 'iso-8859-1']
+        new_df = None
+        last_exception = None
+        
+        for encoding in encodings_to_try:
+            try:
+                csv_content = content.decode(encoding)
+                new_df = pd.read_csv(io.StringIO(csv_content))
+                logger.log_message(f"Successfully read CSV with encoding: {encoding}", level=logging.INFO)
+                break
+            except Exception as e:
+                last_exception = e
+                logger.log_message(f"Failed to read CSV with encoding {encoding}: {str(e)}", level=logging.WARNING)
+                continue
+        
+        if new_df is None:
+            raise HTTPException(status_code=400, detail=f"Error reading file with tried encodings: {encodings_to_try}. Last error: {str(last_exception)}")
+        
+        # Clean and validate the name
+        name = file.filename.replace('.csv', '').replace(' ', '_').lower().strip()
+        
+        # Validate name length and create safe variable name
+        if len(name) > 30:
+            name = name[:30]
+        
+        # Ensure it's a safe Python identifier
+        if not is_safe_variable_name(name):
+            import random
+            name = f"{name}_{random.randint(1000, 9999)}"
+        
+        # Format the description
+        desc = f" exact_python_name: `{name}` Dataset: {file.filename}"
+        
+        # Create datasets dictionary with the new dataset
+        datasets = {name: new_df}
+        
+        # Update the session with the new dataset (this will replace any existing datasets)
+        
+        logger.log_message(f"Successfully previewed dataset '{name}'", level=logging.INFO)
+        
+        return {
+            "headers": new_df.columns.tolist(),
+            "rows": new_df.head(10).values.tolist(),
+            "name": name,
+            "description": desc
+        }
+    except Exception as e:
+        logger.log_message(f"Error in preview_csv_upload: {str(e)}", level=logging.ERROR)
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.post("/generate-session")
+async def generate_session():
+    """Generate a new session ID and initialize it with default dataset"""
+    try:
+        import uuid
+        session_id = str(uuid.uuid4())
+        
+        # Initialize the session with default dataset
+        # This will be handled by the first request to any endpoint that uses get_session_id_dependency
+        
+        logger.log_message(f"Generated new session ID: {session_id}", level=logging.INFO)
+        
+        return {
+            "session_id": session_id,
+            "message": "Session created successfully"
+        }
+    except Exception as e:
+        logger.log_message(f"Error generating session: {str(e)}", level=logging.ERROR)
+        raise HTTPException(status_code=500, detail=f"Failed to generate session: {str(e)}")

src/utils/dataset_description_generator.py

@@ -0,0 +1,79 @@
+import logging
+import pandas as pd
+from typing import Dict
+
+from src.agents.agents import dataset_description_agent, data_context_gen
+from src.utils.model_registry import mid_lm
+from src.utils.logger import Logger
+import dspy
+
+# Initialize logger
+logger = Logger("dataset_description_generator", see_time=False, console_log=False)
+
+def generate_dataset_description(datasets: Dict[str, pd.DataFrame], existing_description: str = "", dataset_names: list = None) -> str:
+    """
+    Generate AI-powered description for datasets
+    
+    Args:
+        datasets: Dictionary of dataset names to DataFrames
+        existing_description: Existing description to improve upon (optional)
+        dataset_names: List of dataset names to use in the description format (optional)
+        
+    Returns:
+        Generated description string with proper exact_python_name formatting
+    """
+    try:
+        if not datasets or len(datasets) == 0:
+            return existing_description
+        
+        # Build dataset view for description generation
+        dataset_view = ""
+        count = 0
+        for table_name, table_df in datasets.items():
+            head_data = table_df.head(3)
+            columns = [{col: str(head_data[col].dtype)} for col in head_data.columns]
+            dataset_view += f"exact_table_name={table_name}\n:columns:{str(columns)}\n{head_data.to_markdown()}\n"
+            count += 1
+        
+        # Generate description using AI
+        with dspy.context(lm=mid_lm):
+            if count == 1:
+                data_context = dspy.Predict(dataset_description_agent)(
+                    existing_description=existing_description,
+                    dataset=dataset_view
+                )
+                generated_desc = data_context.description
+            elif count > 1:
+                data_context = dspy.Predict(data_context_gen)(
+                    user_description=existing_description,
+                    dataset_view=dataset_view
+                )
+                generated_desc = data_context.data_context
+            else:
+                generated_desc = existing_description
+        
+        # Format the description with exact_python_name for all datasets
+        if dataset_names and len(dataset_names) > 0:
+            if len(dataset_names) == 1:
+                # Single dataset format
+                formatted_desc = f" exact_python_name: `{dataset_names[0]}` Dataset: {generated_desc}"
+            else:
+                # Multiple datasets format - list all dataset names
+                names_list = ", ".join([f"`{name}`" for name in dataset_names])
+                formatted_desc = f" exact_python_name: {names_list} Dataset: {generated_desc}"
+        else:
+            # Fallback to original format if no dataset names provided
+            dataset_keys = list(datasets.keys())
+            if len(dataset_keys) == 1:
+                formatted_desc = f" exact_python_name: `{dataset_keys[0]}` Dataset: {generated_desc}"
+            else:
+                names_list = ", ".join([f"`{name}`" for name in dataset_keys])
+                formatted_desc = f" exact_python_name: {names_list} Dataset: {generated_desc}"
+        
+        logger.log_message(f"Successfully generated dataset description for {count} dataset(s)", level=logging.INFO)
+        return formatted_desc
+        
+    except Exception as e:
+        logger.log_message(f"Failed to generate dataset description: {str(e)}", level=logging.WARNING)
+        # Return existing description if generation fails
+        return existing_description

src/utils/model_registry.py

@@ -12,7 +12,7 @@ max_tokens = int(os.getenv("MAX_TOKENS", 6000))
 
 small_lm = dspy.LM('openai/gpt-4o-mini',max_tokens=300,api_key=os.getenv("OPENAI_API_KEY"), cache=False)
 
-mid_lm = dspy.LM('openai/gpt-4o-mini',max_tokens=1000,api_key=os.getenv("OPENAI_API_KEY"), cache=False)
+mid_lm = dspy.LM('openai/gpt-4o-mini',max_tokens=1300,api_key=os.getenv("OPENAI_API_KEY"), cache=False)
 
 gpt_4o_mini = dspy.LM('openai/gpt-4o-mini',max_tokens=4000,api_key=os.getenv("OPENAI_API_KEY"), cache=False)