commit

controller.py

@@ -37,12 +37,14 @@ import uvicorn
 from csv_analysis_service import execute_analysis_logic
 from csv_chart_service import execute_python_code
 from csv_metadata_service import CsvDataRequest, CsvInfoRequest, CsvInfoResponse, PythonExecutionRequest, PythonExecutionResponse, execute_python_logic, get_csv_basic_info, get_robust_csv_rows
+from extract_csv_metadata_service import extract_csv_metadata_logic
 from mongo_service import convert_oid, execute_mongo_operation, extract_database_name, sanitize_json_input
 from pydantic_csv_analysis_model import AnalysisRequest, AnalysisResponse
 from pydantic_csv_charts_model import ChartExecutionPayload, ChartExecutionResponse
 from pydantic_mongo_executor_model import ExecutorPayload, ExecutorResponse
 from report_service import FileBoxProps, ReportRequest, execute_report_generation
 from supabase_service import upload_bytes_to_supabase
+from table_info_model import CsvFieldsRequest, CsvFieldsResponse
 
 # --- Configuration & Setup ---
 load_dotenv()
@@ -631,6 +633,15 @@ async def root():
 async def ping():
     return {"message": "I am alive!"}
 
+@app.post("/api/get_csv_fields", response_model=CsvFieldsResponse)
+async def get_csv_fields_endpoint(payload: CsvFieldsRequest, token: str = Depends(validate_token)):
+    request_id = str(uuid.uuid4())[:8]
+    try:
+        fields = await run_in_threadpool(extract_csv_metadata_logic, csv_url=payload.csv_url)
+        return CsvFieldsResponse(success=True, fields=fields, request_id=request_id)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail={"success": False, "error": str(e), "request_id": request_id})
+
 if __name__ == "__main__":
     host = os.getenv("HOST", "0.0.0.0")
     port = int(os.getenv("PORT", 7860))

extract_csv_metadata_service.py

@@ -0,0 +1,92 @@
+import logging
+from typing import List, Dict, Any, Optional
+import pandas as pd
+import numpy as np
+
+# Configure Logging
+logging.basicConfig(
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    level=logging.INFO
+)
+logger = logging.getLogger("csv_metadata_service")
+
+def extract_csv_metadata_logic(csv_url: str) -> List[Dict[str, Any]]:
+    """
+    Analyzes a CSV file from a URL to infer column metadata.
+    
+    Args:
+        csv_url (str): The public URL of the CSV file.
+        
+    Returns:
+        List[Dict[str, Any]]: A list of dictionaries containing field metadata.
+    """
+    try:
+        logger.info(f"Extracting metadata from: {csv_url}")
+        
+        # Load a sample of the CSV to infer types (first 50 rows is usually enough)
+        # Using on_bad_lines='skip' to be robust against malformed rows
+        # storage_options={'User-Agent': ...} can be added if requests are blocked
+        df = pd.read_csv(csv_url, nrows=50, on_bad_lines='skip') 
+        
+        fields = []
+        
+        for col in df.columns:
+            dtype = df[col].dtype
+            
+            # 1. Default type
+            generic_type = 'string'
+            
+            # 2. Map Pandas/Numpy types to generic keys
+            if pd.api.types.is_integer_dtype(dtype):
+                generic_type = 'integer'
+            elif pd.api.types.is_float_dtype(dtype):
+                generic_type = 'decimal'
+            elif pd.api.types.is_bool_dtype(dtype):
+                generic_type = 'boolean'
+            elif pd.api.types.is_datetime64_any_dtype(dtype):
+                generic_type = 'date'
+            else:
+                # 3. Attempt to detect dates in object/string columns
+                # We check the first non-null value to see if it parses as a date
+                if len(df) > 0:
+                    try:
+                        first_valid = df[col].dropna().iloc[0] if not df[col].dropna().empty else ""
+                        val_str = str(first_valid).strip()
+                        
+                        # Simple check to avoid treating plain numbers as dates
+                        if val_str and not val_str.isdigit():
+                            pd.to_datetime(val_str)
+                            generic_type = 'date'
+                    except (ValueError, TypeError):
+                        # Not a date, keep as string
+                        pass
+
+            # 4. Determine Nullability
+            is_nullable = bool(df[col].isnull().any())
+            
+            # 5. Heuristic for Primary Key
+            lower_name = col.lower()
+            potential_names = ['id', 'uuid', '_id', 'pk']
+            
+            # It's likely a PK if it has a common ID name OR ends in _id
+            is_pk_name = lower_name in potential_names or lower_name.endswith('_id')
+            
+            # It must also be unique in our sample
+            is_unique = df[col].is_unique if not df.empty else False
+            
+            is_primary_key = is_pk_name and is_unique
+
+            fields.append({
+                "name": col,
+                "type": generic_type,
+                "nullable": is_nullable,
+                "isPrimaryKey": is_primary_key
+            })
+            
+        logger.info(f"Successfully extracted {len(fields)} fields.")
+        return fields
+
+    except Exception as e:
+        logger.error(f"CSV Analysis Error for {csv_url}: {str(e)}")
+        # Re-raise to be handled by the controller's exception handler
+        raise e

table_info_model.py

@@ -0,0 +1,18 @@
+from typing import List, Optional
+from pydantic import BaseModel
+
+
+class CsvFieldsRequest(BaseModel):
+    csv_url: str
+
+class FieldMetadata(BaseModel):
+    name: str
+    type: str
+    nullable: bool
+    isPrimaryKey: bool
+
+class CsvFieldsResponse(BaseModel):
+    success: bool
+    fields: List[FieldMetadata]
+    error: Optional[str] = None
+    request_id: str