Modularize my Python code to be used in production and save as a zip file to download and include:
Clear Layer Separation:
    Has folders for api, core, models, services, and utils, indicating solid separation of concerns.
    api/endpoints/ separates route logic by functionality (compliance.py, policy.py), improving maintainability.
Reusability & Extensibility:
    Utility functions are separated into utils/ (e.g., query_builder.py, text_extractor.py).
    Services (like crew_service.py) are abstracted, aiding testability and reuse.
Modular Components:
    The use of models/schemas.py for Pydantic models is industry-standard.
    The project provides more granular modules (e.g., nlp_processor.py, document_loader.py), which encourages modular development.
Also inlcude fetarues: Folder structure, Endpoint separation, Utility functions, Pydantic models, Production readiness, CrewAI logic separation, Documentation. 
Also create and fill in requirement.txt and README, .env files. Also comment each line of code.


#!/usr/bin/env python                  # Shebang for running as an executable script
# coding: utf-8                        # Encoding declaration for UTF-8

# --- Import required packages ---
import os                              # For OS-level operations (env, paths)
import logging                         # To enable logging throughout the app
import re                              # Regex support for pattern matching
import json                            # To handle JSON serialization/deserialization
from fastapi import FastAPI, HTTPException, Depends, Query   # FastAPI web framework primitives
from pydantic import BaseModel, Field   # Data validation and parsing
from typing import List, Dict, Any, Optional  # Type hints for later code
from dotenv import load_dotenv          # Utility for loading .env files into environment variables

# Adding advanced NLP packages for better policy text analysis
import spacy                            # SpaCy NLP package
import nltk                             # NLTK NLP package
from nltk.tokenize import word_tokenize, sent_tokenize  # Tokenizers

from langchain_community.utilities.sql_database import SQLDatabase   # SQL database utility (LangChain)
from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader   # PDF and DOCX loaders
from langchain.text_splitters import RecursiveCharacterTextSplitter  # Functionality to split text into chunks
from langchain.embeddings.openai import OpenAIEmbeddings  # OpenAI embeddings
from langchain.chat_models import ChatOpenAI               # OpenAI chat models
from langchain_community.vectorstores import Chroma        # Vector store for document search
from langchain.tools.retriever.retriever_tool import create_retriever_tool  # Tool creator for retrieval
from langchain_community.agent_toolkits import SQLDatabaseToolkit           # Toolkit for SQL tools
from langchain.tools import tool                           # Decorator for registering tools

from crewai import Agent, Task, Crew, Process              # CrewAI library for multi-agent orchestration

# --- Logging Setup ---
logging.basicConfig(
    level=logging.INFO,                                    # Set log level to INFO
    format="%(asctime)s [%(levelname)s] %(message)s"       # Log output format
)
logger = logging.getLogger(__name__)                       # Logger instance for this module

# --- Load environment variables ---
load_dotenv()                                              # Load .env file into os.environ

# --- Configuration ---
openai_api_key = os.getenv("OPENAI_API_KEY")               # OpenAI API key from environment
if not openai_api_key:
    raise ValueError("OPENAI_API_KEY is missing. Check your .env file.")    # Abort if missing

db_user = os.getenv("DB_USER", "")                         # Database username
db_password = os.getenv("DB_PASSWORD", "")                 # Database password
db_host = os.getenv("DB_HOST", "")                         # Database host
db_name = os.getenv("DB_NAME", "classicmodels")            # Database name (default: classicmodels)

folder_path = os.getenv("DOCS_PATH", "./docs")             # Documents folder path

# --- Initialize FastAPI ---
app = FastAPI(                                             # Create FastAPI app instance
    title="Policy Compliance API",
    description="API for checking policy compliance using NLP and database analysis",
    version="1.0.0"
)

# --- Initialize NLP components ---
nlp = None                                                     # SpaCy NLP instance placeholder
try:
    nltk.download('punkt', quiet=True)                         # Download punkt for NLTK tokenization
    if not spacy.util.is_package('en_core_web_sm'):            # Ensure SpaCy English model is installed
        logger.info("Downloading Spacy model...")
        spacy.cli.download('en_core_web_sm')
    nlp = spacy.load("en_core_web_sm")                         # Load SpaCy English model
except Exception as e:
    logger.error(f"Error initializing NLP components: {e}")    # Log startup errors

# --- Define Pydantic Models ---
# Used for request and response validation in FastAPI
class ComplianceQuery(BaseModel):                              # Input model for compliance checks
    query_text: str = Field(..., description="Natural language query about policy compliance")
    department: Optional[str] = Field(None, description="Optional department filter")
    
class PolicyViolation(BaseModel):                              # Model for a single policy violation
    employee_id: str
    first_name: str
    last_name: str
    department: str
    violation_type: str
    policy_reference: str
    
class ComplianceResponse(BaseModel):                           # Output model for compliance results
    query: str
    violations: List[PolicyViolation]
    total_count: int
    relevant_policies: List[str]

class PolicySearchQuery(BaseModel):                            # Input model for document search
    search_text: str = Field(..., description="Text to search for in policy documents")

class PolicySearchResult(BaseModel):                           # Output model for each policy search result
    document_name: str
    content: str
    relevance_score: float

class PolicySearchResponse(BaseModel):                         # Output model for policy search results
    query: str
    results: List[PolicySearchResult]

# --- Define SQL Query Templates ---
SQL_QUERY_TEMPLATES = {                       # Templates for parameterized SQL
    "violation_count": "SELECT COUNT(*) FROM employees WHERE ?;",
    "salary_violation": "SELECT * FROM employees WHERE salary < ?;",
    "policy_breach": "SELECT employee_name FROM violations WHERE policy_id IN (SELECT id FROM policies WHERE criteria = ?);",
    "detailed_violations": "SELECT employee_id, first_name, last_name FROM employees WHERE ?;",
    "department_summary": "SELECT department, COUNT(*) as violation_count FROM employees WHERE ? GROUP BY department;"
}

# --- Load and Split Documents ---
def load_documents(folder_path: str) -> List:                 # Loads policy documents from folder
    """Loads PDF and DOCX files from the specified folder."""
    documents = []
    if not os.path.exists(folder_path):                       # If folder doesn't exist, warn and return empty
        logger.warning(f"Folder does not exist: {folder_path}")
        return documents
    for filename in os.listdir(folder_path):                  # Scan each file in the folder
        file_path = os.path.join(folder_path, filename)
        try:
            if filename.endswith('.pdf'):                     # PDF loading
                documents.extend(PyPDFLoader(file_path).load())
            elif filename.endswith('.docx'):                  # DOCX loading
                documents.extend(Docx2txtLoader(file_path).load())
            else:
                logger.info(f"Unsupported file type: {filename}")  # Skip unsupported files
        except Exception as e:
            logger.error(f"Error loading {filename}: {e}")        # Log loading errors
    return documents

# Load documents at startup
logger.info("Loading policy documents...")
documents = load_documents(folder_path)

if not documents:                                         # Fill with default doc if none loaded
    logger.warning("No documents loaded! Vector store may be empty.")
    documents = [{"page_content": "No policy documents found. Please add documents to the docs folder.", "metadata": {"source": "system"}}]

text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)   # Splitter for text chunking
splits = text_splitter.split_documents(documents)                                   # Split all docs into smaller chunks

logger.info("Creating embeddings and vector store...")
embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)                        # Create embeddings object for semantic search
vectorstore = Chroma.from_documents(splits, embedding=embeddings)                   # Build vector store for fast vector search
retriever = vectorstore.as_retriever()                                              # Make as retriever

policy_retriever_tool = create_retriever_tool(
    retriever,
    "policy_document_retriever",
    "Searches and returns policy document information."
)

# --- Database Setup ---
# Try to connect to database
logger.info(f"Connecting to database {db_name}@{db_host}")
db = None
db_tools = []
try:
    connection_string = f"mysql+pymysql://{db_user}:{db_password}@{db_host}/{db_name}"  # Compose DB string
    db = SQLDatabase.from_uri(connection_string)         # Create SQL database interface
    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, openai_api_key=openai_api_key) # Create LLM for toolkits
    db_toolkit = SQLDatabaseToolkit(db=db, llm=llm)      # Toolkit for utilizing the DB
    db_tools = db_toolkit.get_tools()                    # Extract tools from toolkit
    logger.info("Database connection successful")
except Exception as e:
    logger.error(f"Database connection failed: {e}")     # Log on failure
    if "Access denied" in str(e):                        # Granular error messages
        logger.error("Database credentials are incorrect")
    elif "Unknown database" in str(e):
        logger.error(f"Database '{db_name}' does not exist")
    elif "Can't connect" in str(e):
        logger.error(f"Cannot connect to host '{db_host}'")

# --- Helper function for enhanced criteria mapping ---
def extract_criteria_from_text(text: str) -> Dict[str, Any]:
    """
    Enhanced criteria extraction using regex, heuristics, and NLP.
    Combines pattern matching with NLP capabilities.
    """
    criteria = {}

    # Basic regex patterns
    hire_date_match = re.search(r"employees hired (before|after) (\d{4})", text, re.IGNORECASE) # Extract hire date
    if hire_date_match:
        operator = "<" if hire_date_match.group(1).lower() == "before" else ">"
        year = hire_date_match.group(2)
        criteria["hire_date"] = {"operator": operator, "value": f"{year}-01-01"}

    dept_match = re.search(r"department[s]? (is|are|in) ([\w\s,]+)", text, re.IGNORECASE) # Extract department(s)
    if dept_match:
        depts = [d.strip() for d in dept_match.group(2).split(',')]
        criteria["department"] = {"operator": "IN", "values": depts}
    
    # Enhanced NLP-based extraction
    if nlp:
        doc = nlp(text)                               # Analyze text via SpaCy NLP
        
        # Extract salary information
        salary_pattern = re.search(r"salary (below|under|less than|above|over|more than) (\d+[k]?)", text, re.IGNORECASE)
        if salary_pattern:                            # Parse salary constraint
            comparison = salary_pattern.group(1).lower()
            amount = salary_pattern.group(2)
            if amount.endswith('k'):                  # Convert if needed
                amount = int(amount[:-1]) * 1000
            else:
                amount = int(amount)
            operator = "<" if comparison in ["below", "under", "less than"] else ">"
            criteria["salary"] = {"operator": operator, "value": amount}
            
        # Extract date ranges using SpaCy NER
        for ent in doc.ents:
            if ent.label_ == "DATE":
                date_text = ent.text
                if "since" in text[:ent.start].lower() or "after" in text[:ent.start].lower():
                    criteria["date_range"] = {"operator": ">", "value": date_text}
                elif "before" in text[:ent.start].lower() or "until" in text[:ent.start].lower():
                    criteria["date_range"] = {"operator": "<", "value": date_text}
    
    return criteria                                 # Return extracted query criteria

# --- Function to safely build SQL queries ---
def build_safe_sql_query(template: str, criteria: Dict[str, Any]) -> tuple:
    """
    Builds a safe SQL query using parameterized queries to prevent SQL injection.
    Returns the query string and parameters tuple.
    """
    params = []
    conditions = []
    
    for key, value in criteria.items():                 # Build WHERE clause by criteria
        if key == "hire_date":
            conditions.append(f"hire_date {value['operator']} %s")
            params.append(value['value'])
        elif key == "department":
            placeholders = ", ".join(["%s"] * len(value['values']))
            conditions.append(f"department IN ({placeholders})")
            params.extend(value['values'])
        elif key == "salary":
            conditions.append(f"salary {value['operator']} %s")
            params.append(value['value'])
        elif key == "date_range":
            conditions.append(f"event_date {value['operator']} %s")
            params.append(value['value'])
    
    where_clause = " AND ".join(conditions) if conditions else "1=1"     # Combine, or always true if none
    query = template.replace("?", where_clause)                          # Insert into template
    
    return query, tuple(params)                                          # Return query and args

# --- CrewAI Implementation ---
def setup_crew():
    """
    Sets up the CrewAI agents and tasks for policy analysis.
    """
    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0, openai_api_key=openai_api_key)
    
    # Create policy analyst agent
    policy_analyst = Agent(
        role="Policy Analyst",
        goal="Analyze policy documents and extract key compliance requirements",
        backstory="You are an expert in corporate policy analysis with years of experience in compliance",
        verbose=True,
        llm=llm,
        tools=[policy_retriever_tool]
    )
    
    # Create data analyst agent
    data_analyst = Agent(
        role="Data Analyst",
        goal="Query databases to find compliance violations based on policy requirements",
        backstory="You are a skilled data analyst who specializes in finding patterns in employee data",
        verbose=True,
        llm=llm,
        tools=db_tools
    )
    
    # Create compliance officer agent
    compliance_officer = Agent(
        role="Compliance Officer",
        goal="Evaluate policy violations and provide recommendations",
        backstory="You are a senior compliance officer responsible for ensuring company-wide policy adherence",
        verbose=True,
        llm=llm
    )
    
    # Create tasks for the agents
    policy_analysis_task = Task(
        description="Analyze policy documents to extract compliance requirements",
        agent=policy_analyst,
        expected_output="A list of key compliance requirements from policy documents"
    )
    
    data_analysis_task = Task(
        description="Query the database to find potential policy violations",
        agent=data_analyst,
        expected_output="A list of potential policy violations with employee details"
    )
    
    compliance_evaluation_task = Task(
        description="Evaluate violations and provide recommendations for compliance improvement",
        agent=compliance_officer,
        expected_output="A comprehensive compliance report with recommendations"
    )
    
    # Create and return crew (takes agents and tasks)
    crew = Crew(
        agents=[policy_analyst, data_analyst, compliance_officer],
        tasks=[policy_analysis_task, data_analysis_task, compliance_evaluation_task],
        verbose=2,
        process=Process.sequential
    )
    return crew

# --- API Endpoints ---
@app.get("/")
async def root():
    """Root endpoint that returns API information."""
    return {
        "message": "Policy Compliance API",
        "version": "1.0.0",
        "status": "operational",
        "endpoints": ["/compliance/check", "/policy/search"]
    }

@app.post("/compliance/check", response_model=ComplianceResponse)
async def check_compliance(query: ComplianceQuery):
    """
    Checks policy compliance based on natural language query.
    Uses CrewAI to analyze policies and find violations.
    """
    if not db:
        raise HTTPException(status_code=503, detail="Database connection not available")
    
    try:
        criteria = extract_criteria_from_text(query.query_text)         # NLP extract relevant criteria from user query
        if query.department:
            criteria["department"] = {"operator": "=", "values": [query.department]} # Handle department filter
        
        template = SQL_QUERY_TEMPLATES["detailed_violations"]
        sql_query, params = build_safe_sql_query(template, criteria)    # Safe SQL creation
        
        # (In production, should run CrewAI with these inputs, here just mock for demo)
        violations = [
            PolicyViolation(
                employee_id="E001",
                first_name="John",
                last_name="Doe",
                department="Sales",
                violation_type="Salary Policy",
                policy_reference="Section 3.2 of Compensation Policy"
            ),
            PolicyViolation(
                employee_id="E002",
                first_name="Jane",
                last_name="Smith",
                department="Marketing",
                violation_type="Work Hours",
                policy_reference="Section 2.1 of Employee Handbook"
            )
        ]
        
        # Use semantic retriever to find relevant policies for query
        policy_docs = retriever.get_relevant_documents(query.query_text)
        relevant_policies = [doc.metadata.get("source", "Unknown") for doc in policy_docs[:3]]
        
        return ComplianceResponse(
            query=query.query_text,
            violations=violations,
            total_count=len(violations),
            relevant_policies=relevant_policies
        )
    except Exception as e:
        logger.error(f"Error processing compliance check: {e}")
        raise HTTPException(status_code=500, detail=f"Error processing request: {str(e)}")

@app.post("/policy/search", response_model=PolicySearchResponse)
async def search_policy(query: PolicySearchQuery):
    """
    Searches policy documents for relevant information.
    """
    try:
        docs = retriever.get_relevant_documents(query.search_text)            # Get up-to-5 relevant docs
        results = []
        for i, doc in enumerate(docs[:5]):
            results.append(
                PolicySearchResult(
                    document_name=doc.metadata.get("source", "Unknown"),
                    content=doc.page_content[:500] + "..." if len(doc.page_content) > 500 else doc.page_content,
                    relevance_score=1.0 - (i * 0.1)  # Mock relevance score
                )
            )
        return PolicySearchResponse(
            query=query.search_text,
            results=results
        )
    except Exception as e:
        logger.error(f"Error searching policy documents: {e}")
        raise HTTPException(status_code=500, detail=f"Error processing request: {str(e)}")

# --- Main execution (for running as script) ---
if __name__ == "__main__":
    import uvicorn
    
    if db:
        logger.info("Setting up CrewAI...")
        crew = setup_crew()                                # Set up agents and workflow
    
    logger.info("Starting API server...")
    uvicorn.run(app, host="0.0.0.0", port=8000)            # Launch FastAPI on all interfaces