Upload 8 files

.streamlit/config.toml

@@ -0,0 +1,55 @@
+[server]
+enableStaticServing = true
+
+[[theme.fontFaces]]
+family = "SpaceGrotesk"
+url = "app/static/SpaceGrotesk-VariableFont_wght.ttf"
+
+[[theme.fontFaces]]
+family = "SpaceMono"
+url = "app/static/SpaceMono-Bold.ttf"
+style = "normal"
+weight = 700
+
+[[theme.fontFaces]]
+family = "SpaceMono"
+url = "app/static/SpaceMono-BoldItalic.ttf"
+style = "italic"
+weight = 700
+
+[[theme.fontFaces]]
+family = "SpaceMono"
+url = "app/static/SpaceMono-Italic.ttf"
+style = "italic"
+weight = 400
+
+[[theme.fontFaces]]
+family = "SpaceMono"
+url = "app/static/SpaceMono-Regular.ttf"
+style = "normal"
+weight = 400
+
+[theme]
+primaryColor = "#cb785c"
+backgroundColor = "#fdfdf8"
+secondaryBackgroundColor = "#ecebe3"
+textColor = "#3d3a2a"
+linkColor = "#3d3a2a"
+borderColor = "#d3d2ca"
+showWidgetBorder = true
+baseRadius = "0.75rem"
+buttonRadius = "full"
+font = "SpaceGrotesk"
+headingFontWeights = [600,500,500,500,500,500]
+headingFontSizes = ["3rem", "2rem"]
+codeFont = "SpaceMono"
+codeFontSize = ".75rem"
+codeBackgroundColor = "#ecebe4"
+showSidebarBorder = true
+chartCategoricalColors = ["#0ea5e9", "#059669", "#fbbf24"]
+
+[theme.sidebar]
+backgroundColor = "#f0f0ec"
+secondaryBackgroundColor = "#ecebe3"
+headingFontSizes = ["1.6rem", "1.4rem", "1.2rem"]
+dataframeHeaderBackgroundColor = "#e4e4e0"

Dockerfile

@@ -0,0 +1,32 @@
+# Use official Python image as base
+FROM python:3.10-slim
+
+# Set working directory
+WORKDIR /app
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    STREAMLIT_SERVER_PORT=8501 \
+    STREAMLIT_SERVER_HEADLESS=true \
+    STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first to leverage Docker cache
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application
+COPY . .
+
+# Expose the port Streamlit runs on
+EXPOSE 8501
+
+# Command to run the application
+CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"]

README.md

@@ -1,11 +1,146 @@
----
-title: Role Base Access Control
-emoji: 🐨
-colorFrom: purple
-colorTo: yellow
-sdk: docker
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Departmental RAG System with Role-Based Access Control
+
+A production-ready Streamlit-based RAG (Retrieval-Augmented Generation) application with MongoDB authentication and role-based access control. This application allows users to securely access and query department-specific documents based on their assigned roles.
+
+![Image](https://github.com/user-attachments/assets/f8a91110-a40f-4f7f-8c76-43fd0932e10c)
+
+## ✨ Features
+
+- **Secure Authentication**: MongoDB-based user authentication
+- **Role-Based Access Control (RBAC)**: Granular access to department-specific documents
+- **Efficient Document Retrieval**: ChromaDB vector store with Cohere embeddings
+- **Natural Language Queries**: Powered by Groq's LLM with Cohere embeddings
+- **Modern UI**: Clean, responsive interface with dark/light theme support
+- **Department-Specific Knowledge Bases**: Isolated document repositories for each department
+
+## 🛠️ Prerequisites
+
+- Python 3.8+
+- MongoDB Atlas account (or local MongoDB instance)
+- Cohere API key (for text embeddings)
+- Groq API key (for LLM, optional but recommended)
+
+## 🚀 Quick Start
+
+1. **Clone the repository**
+   ```bash
+   git clone https://github.com/Bharath8080/RBAC_RAG_V4.git
+   cd RBAC_RAG_V4
+   ```
+
+2. **Set up the environment**
+   ```bash
+   # Install uv (if not already installed)
+   pip install uv
+   
+   # Create and activate virtual environment
+   uv venv
+   .venv\Scripts\activate  # On Windows
+   # OR
+   source .venv/bin/activate  # On Unix/macOS
+   
+   # Install dependencies
+   uv pip install -r requirements.txt
+   ```
+
+3. **Configure environment variables**
+   Create a `.env` file in the project root with the following content:
+   ```
+   # MongoDB Configuration
+   MONGO_URI=mongodb+srv://<username>:<password>@<cluster-address>/<database>?retryWrites=true&w=majority
+   DB_NAME=rag_system
+
+   # Cohere API Key (for embeddings)
+   COHERE_API_KEY=your_cohere_api_key_here
+
+   # Optional: Groq API Key (for LLM)
+   GROQ_API_KEY=your_groq_api_key_here
+   ```
+   Replace the placeholder values with your actual credentials.
+
+4. **Set up the vector store**
+   ```bash
+   python ingest.py
+   ```
+   This will process and index all documents in the `resources/data/` directory.
+
+5. **Initialize the database with sample users**
+   ```bash
+   python ingest_db.py
+   ```
+   This will create sample user accounts in your MongoDB database.
+
+6. **Run the application**
+   ```bash
+   streamlit run app.py
+   ```
+
+7. **Access the application**
+   Open your browser and navigate to `http://localhost:8501`
+
+## 👥 Default User Accounts
+
+| Username | Password    | Department  |
+|----------|-------------|-------------|
+| tony     | password123 | engineering |
+| bruce    | securepass  | marketing   |
+| sam      | financepass | finance     |
+| natasha  | hrpass123   | hr          |
+
+## 📁 Project Structure
+
+```
+RBAC_RAG_V4/
+├── .env.example           # Example environment variables
+├── app.py                # Main Streamlit application
+├── database.py           # Database connection and user management
+├── ingest.py             # Document processing and vector store creation
+├── ingest_db.py          # Initialize MongoDB with sample users
+├── requirements.txt      # Python dependencies
+├── resources/
+│   └── data/             # Department-specific documents
+│       ├── engineering/
+│       ├── finance/
+│       ├── hr/
+│       └── marketing/
+└── chroma_db/            # ChromaDB vector store (created after first run)
+```
+
+## 🔧 Configuration
+
+### Environment Variables
+
+- `MONGO_URI`: MongoDB connection string
+- `DB_NAME`: Database name (default: `rag_system`)
+- `COHERE_API_KEY`: Required for text embeddings
+- `GROQ_API_KEY`: Required for LLM responses (falls back to local model if not provided)
+
+### Adding New Users
+
+1. Run the `ingest_db.py` script with new user data
+2. Or add users directly to MongoDB in the `users` collection
+
+### Adding New Documents
+
+1. Place documents in the appropriate department folder under `resources/data/`
+2. Run `python ingest.py` to update the vector store
+
+## 🛡️ Security Notes
+
+- Always store sensitive information in environment variables, never in code
+- Use strong, unique passwords for MongoDB access
+- Regularly update your dependencies for security patches
+- Consider implementing rate limiting in production
+- Enable MongoDB network access restrictions
+
+## 📄 License
+
+This project is part of the Codebasics Resume Project Challenge.
+
+## 🙏 Acknowledgments
+
+- [Streamlit](https://streamlit.io/) for the web framework
+- [ChromaDB](https://www.trychroma.com/) for vector storage
+- [Cohere](https://cohere.com/) for embeddings
+- [Groq](https://groq.com/) for LLM inference
+- [MongoDB](https://www.mongodb.com/) for user management

app.py

@@ -0,0 +1,321 @@
+__import__('pysqlite3')
+import sys
+sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
+
+import sqlite3
+
+import os
+import streamlit as st
+import chromadb
+from typing import Dict, Optional, Any
+from pathlib import Path
+from dotenv import load_dotenv
+from llama_index.core import VectorStoreIndex, StorageContext, Settings
+from llama_index.vector_stores.chroma import ChromaVectorStore
+from llama_index.llms.groq import Groq
+from llama_index.embeddings.cohere import CohereEmbedding
+
+
+from arize.otel import register
+from openinference.instrumentation.llama_index import LlamaIndexInstrumentor
+
+# Setup OTel via Arize's convenience function
+tracer_provider = register(
+    space_id=os.getenv("ARIZE_SPACE_ID"),
+    api_key=os.getenv("ARIZE_API_KEY"),
+    project_name="rbacrag" # Choose a project name
+)
+
+# Instrument LlamaIndex
+LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider)
+
+# Import database module
+from database import db, initialize_users
+
+# Load environment variables
+load_dotenv()
+
+# Initialize default users
+initialize_users()
+
+# Role-based access control for documents
+ROLE_ACCESS = {
+    "hr": ["hr", "general"],
+    "engineering": ["engineering", "general"],
+    "finance": ["finance", "general"],
+    "marketing": ["marketing", "general"]
+}
+
+# Initialize session state
+def initialize_session_state():
+    """Initialize or reset the session state"""
+    if "authenticated" not in st.session_state:
+        st.session_state.authenticated = False
+    if "username" not in st.session_state:
+        st.session_state.username = None
+    if "role" not in st.session_state:
+        st.session_state.role = None
+    if "messages" not in st.session_state:
+        st.session_state.messages = []
+    if "vector_index" not in st.session_state:
+        st.session_state.vector_index = None
+    if "query_engine" not in st.session_state:
+        st.session_state.query_engine = None
+
+# Set page config
+st.set_page_config(
+    page_title="Departmental RAG System",
+    page_icon="🔒",
+    layout="centered",
+    initial_sidebar_state="collapsed"
+)
+
+# Initialize session state
+initialize_session_state()
+
+# Authentication functions
+def login(username: str, password: str) -> bool:
+    """
+    Authenticate user and set session state
+    
+    Args:
+        username: The username to authenticate
+        password: The password to verify
+        
+    Returns:
+        bool: True if authentication was successful, False otherwise
+    """
+    try:
+        user = db.verify_user(username, password)
+        if user:
+            st.session_state.authenticated = True
+            st.session_state.username = user["username"]
+            st.session_state.role = user["role"]
+            st.session_state.messages = [
+                {"role": "assistant", "content": f"Welcome, {user['username']}! How can I assist you today?"}
+            ]
+            st.rerun()  # Rerun to update the UI
+            return True
+        return False
+    except Exception as e:
+        st.error(f"An error occurred during login: {str(e)}")
+        return False
+
+def logout():
+    """
+    Log out the current user and clear session state
+    """
+    username = st.session_state.get('username', 'Unknown')
+    st.session_state.clear()
+    initialize_session_state()
+    st.success(f"Successfully logged out {username}")
+    st.rerun()  # Rerun to update the UI
+
+@st.cache_resource
+def load_vector_index(role: str):
+    """Load the ChromaDB index for the user's role"""
+    try:
+        # Initialize Cohere embeddings
+        cohere_api_key = os.getenv("COHERE_API_KEY")
+        if not cohere_api_key:
+            raise ValueError("COHERE_API_KEY not found in environment variables")
+            
+        embed_model = CohereEmbedding(
+            cohere_api_key=cohere_api_key,
+            model_name="embed-english-v3.0",
+            input_type="search_document"
+        )
+        Settings.embed_model = embed_model
+        
+        # Initialize Chroma client
+        persist_dir = f"./chroma_db/{role}"
+        chroma_client = chromadb.PersistentClient(path=persist_dir)
+        
+        # Get the collection
+        chroma_collection = chroma_client.get_collection("documents")
+        
+        # Create vector store
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        
+        # Create storage context
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        
+        # Load the index
+        index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store,
+            storage_context=storage_context,
+            embed_model=embed_model
+        )
+        
+        return index
+    except Exception as e:
+        st.error(f"Error loading vector index: {str(e)}")
+        st.stop()
+
+def chat_interface():
+    """Main chat interface"""
+    # Add styled heading
+    st.markdown(f"<h2 style='color: #1407fa;'>💬 {st.session_state.role.capitalize()} Department Chat</h3>", unsafe_allow_html=True)
+    
+    # Display chat messages
+    for message in st.session_state.messages:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+    # Load the appropriate index for the user's role
+    index = load_vector_index(st.session_state.role)
+    
+    # Initialize Groq LLM
+    try:
+        llm = Groq(
+            model="llama3-8b-8192", 
+            api_key=os.getenv("GROQ_API_KEY"),
+            temperature=0.5,
+            system_prompt=f"You are a helpful assistant specialized in {st.session_state.role} department documents. Answer the user queries with the help of the provided context with high accuracy and precision."
+        )
+        
+        # Create query engine with the LLM
+        query_engine = index.as_query_engine(
+            llm=llm,
+            similarity_top_k=3,
+            response_mode="compact"
+        )
+    except Exception as e:
+        st.error(f"Error initializing LLM: {str(e)}")
+        st.warning("Falling back to default LLM settings. Some features may be limited.")
+        query_engine = index.as_query_engine(
+            similarity_top_k=3,
+            response_mode="compact"
+        )
+    
+    # Chat input
+    if prompt := st.chat_input(f"Ask about {st.session_state.role} documents..."):
+        # Add user message to chat history
+        st.session_state.messages.append({"role": "user", "content": prompt})
+        
+        # Display user message
+        with st.chat_message("user"):
+            st.markdown(prompt)
+        
+        # Get and display assistant response
+        with st.chat_message("assistant"):
+            message_placeholder = st.empty()
+            full_response = ""
+            
+            try:
+                # Get response from query engine
+                response = query_engine.query(prompt)
+                full_response = str(response)
+                message_placeholder.markdown(full_response)
+            except Exception as e:
+                error_msg = f"Error generating response: {str(e)}"
+                message_placeholder.error(error_msg)
+                full_response = error_msg
+        
+        # Add assistant response to chat history
+        st.session_state.messages.append({"role": "assistant", "content": full_response})
+
+def show_login_form():
+    """Display the beautiful login form"""
+    st.markdown(
+        """
+        <style>
+            .main {
+                background-color: #1a1a2e;
+                color: white;
+            }
+            .stTextInput > div > div > input {
+                background-color: #2a2a3e;
+                color: white;
+                border: 1px solid #4a4a6a;
+                border-radius: 8px;
+            }
+            .stTextInput > div > div > input::placeholder {
+                color: #a0a0b0 !important;
+                opacity: 1 !important;
+            }
+            .stButton > button {
+                background-color: #e94560;
+                color: white;
+                border: none;
+                border-radius: 8px;
+                padding: 10px 20px;
+                font-size: 16px;
+                width: 100%;
+            }
+            .stButton > button:hover {
+                background-color: #d83450;
+            }
+            h1, h2, h3, h4, h5, h6 {
+                color: white;
+            }
+            .st-emotion-cache-1r6slb0 {
+                border: 1px solid #4a4a6a;
+                border-radius: 12px;
+                padding: 2rem;
+                background-color: #232339;
+            }
+        </style>
+        """,
+        unsafe_allow_html=True
+    )
+    st.markdown('<div style="text-align: center; margin-top: -80px; margin-bottom: 30px;"><h1 style="font-size: 3rem;">🔒</h1></div>', unsafe_allow_html=True)
+    st.markdown('<h1 style="text-align: center; margin-bottom: 20px;">Department Portal</h1>', unsafe_allow_html=True)
+    st.markdown('<p style="text-align: center; color: #a0a0b0; margin-bottom: 30px;">Sign in to access your department\'s knowledge base</p>', unsafe_allow_html=True)
+
+    with st.container():
+        with st.form("login_form", border=True):
+            username = st.text_input("Username", placeholder="Enter your username")
+            password = st.text_input("Password", type="password", placeholder="Enter your password")
+            login_button = st.form_submit_button("Sign In")
+            
+            if login_button:
+                if not username or not password:
+                    st.error("Please enter both username and password")
+                elif login(username, password):
+                    st.success(f"Welcome, {username}! Redirecting...")
+                else:
+                    st.error("Invalid username or password")
+
+    with st.expander("Need demo credentials?"):
+        st.markdown("""
+        - **Engineering:** `Tony` / `password123`
+        - **Marketing:** `Bruce` / `securepass`
+        - **Finance:** `Sam` / `financepass`
+        - **HR:** `Natasha` / `hrpass123`
+        """)
+
+    st.markdown('<p style="text-align: center; margin-top: 2rem; color: #a0a0b0;">2025 Department RAG System</p>', unsafe_allow_html=True)
+
+
+def main():
+    """
+    Main application entry point
+    Handles routing between login and main application
+    """
+    # Sidebar for logout and user info
+    if st.session_state.authenticated:
+        st.set_page_config(layout="wide", initial_sidebar_state="expanded")
+        with st.sidebar:
+            st.markdown(f"### Welcome, {st.session_state.username}")
+            st.markdown(f"**Role:** {st.session_state.role.capitalize()}")
+            
+            if st.button("Logout", key="logout_btn"):
+                logout()
+                return
+            
+            st.markdown("---")
+            st.markdown("### About")
+            st.markdown("""
+            This is a secure departmental RAG system that provides
+            role-based access to information across different departments.
+            """)
+    
+    # Main content area
+    if not st.session_state.authenticated:
+        show_login_form()
+    else:
+        chat_interface()
+
+if __name__ == "__main__":
+    main()

database.py

@@ -0,0 +1,161 @@
+import os
+from datetime import datetime
+from typing import Optional, Dict, Any, List
+from pymongo import MongoClient, ReturnDocument
+from pymongo.errors import DuplicateKeyError, ConnectionFailure
+import bcrypt
+from dotenv import load_dotenv
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+class Database:
+    def __init__(self):
+        """Initialize database connection and ensure indexes"""
+        try:
+            mongo_uri = os.getenv("MONGO_URI")
+            if not mongo_uri:
+                raise ValueError("MONGO_URI environment variable is not set")
+                
+            self.client = MongoClient(
+                mongo_uri,
+                serverSelectionTimeoutMS=5000,  # 5 second timeout
+                connectTimeoutMS=30000,        # 30 second connection timeout
+                socketTimeoutMS=45000,          # 45 second socket timeout
+                connect=False                   # Lazy connection
+            )
+            
+            # Test the connection
+            self.client.admin.command('ping')
+            
+            self.db = self.client[os.getenv("DB_NAME", "rag_system")]
+            self.users = self.db["users"]
+            self._create_indexes()
+            logger.info("Successfully connected to MongoDB")
+            
+        except Exception as e:
+            logger.error(f"Failed to connect to MongoDB: {str(e)}")
+            raise
+    
+    def _create_indexes(self):
+        """Create necessary database indexes"""
+        try:
+            # Create unique index on username
+            self.users.create_index("username", unique=True)
+            logger.info("Created database indexes")
+        except Exception as e:
+            logger.error(f"Error creating indexes: {str(e)}")
+            raise
+    
+    def add_user(self, username: str, password: str, role: str) -> bool:
+        """Add a new user to the database"""
+        if not username or not password or not role:
+            logger.warning("Missing required fields for user creation")
+            return False
+            
+        try:
+            hashed = bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt())
+            user_data = {
+                "username": username,
+                "password": hashed.decode('utf-8'),
+                "role": role.lower(),
+                "created_at": datetime.utcnow(),
+                "last_login": None
+            }
+            
+            result = self.users.insert_one(user_data)
+            if result.inserted_id:
+                logger.info(f"Created new user: {username}")
+                return True
+            return False
+            
+        except DuplicateKeyError:
+            logger.warning(f"Username already exists: {username}")
+            return False
+        except Exception as e:
+            logger.error(f"Error adding user {username}: {str(e)}")
+            return False
+    
+    def verify_user(self, username: str, password: str) -> Optional[Dict[str, Any]]:
+        """Verify user credentials"""
+        try:
+            user = self.users.find_one({"username": username})
+            if not user:
+                logger.warning(f"Login attempt for non-existent user: {username}")
+                return None
+                
+            if bcrypt.checkpw(password.encode('utf-8'), user["password"].encode('utf-8')):
+                # Update last login time
+                self.users.update_one(
+                    {"_id": user["_id"]},
+                    {"$set": {"last_login": datetime.utcnow()}}
+                )
+                logger.info(f"Successful login for user: {username}")
+                return {
+                    "username": user["username"],
+                    "role": user["role"],
+                    "last_login": user.get("last_login")
+                }
+            
+            logger.warning(f"Failed login attempt for user: {username}")
+            return None
+            
+        except Exception as e:
+            logger.error(f"Error verifying user {username}: {str(e)}")
+            return None
+    
+    def get_user(self, username: str) -> Optional[Dict[str, Any]]:
+        """Get user by username (without sensitive data)"""
+        try:
+            user = self.users.find_one(
+                {"username": username},
+                {"password": 0}  # Exclude password from results
+            )
+            return user
+        except Exception as e:
+            logger.error(f"Error fetching user {username}: {str(e)}")
+            return None
+
+# Initialize database connection
+db = Database()
+
+def initialize_users():
+    """
+    Initialize default users if they don't exist.
+    Returns tuple of (success_count, total_users, errors)
+    """
+    from datetime import datetime
+    
+    default_users = [
+        {"username": "Tony", "password": "password123", "role": "engineering"},
+        {"username": "Bruce", "password": "securepass", "role": "marketing"},
+        {"username": "Sam", "password": "financepass", "role": "finance"},
+        {"username": "Peter", "password": "pete123", "role": "engineering"},
+        {"username": "Sid", "password": "sidpass123", "role": "marketing"},
+        {"username": "Natasha", "password": "hrpass123", "role": "hr"}
+    ]
+    
+    success_count = 0
+    errors = []
+    
+    for user in default_users:
+        try:
+            if db.add_user(user["username"], user["password"], user["role"]):
+                success_count += 1
+                logger.info(f"Initialized user: {user['username']}")
+            else:
+                errors.append(f"Failed to add user: {user['username']}")
+        except Exception as e:
+            error_msg = f"Error initializing user {user['username']}: {str(e)}"
+            logger.error(error_msg)
+            errors.append(error_msg)
+    
+    logger.info(f"User initialization complete. Success: {success_count}/{len(default_users)}")
+    if errors:
+        logger.warning(f"Encountered {len(errors)} errors during user initialization")
+    
+    return success_count, len(default_users), errors

ingest.py

@@ -0,0 +1,142 @@
+import os
+import chromadb
+from pathlib import Path
+from dotenv import load_dotenv
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, StorageContext, Settings
+from llama_index.vector_stores.chroma import ChromaVectorStore
+from llama_index.embeddings.cohere import CohereEmbedding
+
+# Load environment variables
+load_dotenv()
+
+# Configure the embedding model
+cohere_api_key = os.getenv("COHERE_API_KEY")
+if not cohere_api_key:
+    raise ValueError("COHERE_API_KEY not found in environment variables")
+
+# Initialize the embedding model
+embed_model = CohereEmbedding(
+    cohere_api_key=cohere_api_key,
+    model_name="embed-english-v3.0",
+    input_type="search_document"
+)
+
+# Set the global embedding model
+Settings.embed_model = embed_model
+
+def process_documents(department: str, base_dir: str = "./resources/data"):
+    """
+    Process and index documents for a specific department
+    
+    Args:
+        department: The department name (e.g., 'hr', 'engineering')
+        base_dir: Base directory containing department folders
+    """
+    print(f"Processing documents for {department} department...")
+    
+    # Define paths
+    dept_path = Path(base_dir) / department
+    general_path = Path(base_dir) / "general"
+    persist_dir = f"./chroma_db/{department}"
+    
+    # Create directory if it doesn't exist
+    os.makedirs(persist_dir, exist_ok=True)
+    
+    # Initialize Chroma client
+    chroma_client = chromadb.PersistentClient(path=persist_dir)
+    
+    # Clear existing collection if it exists
+    try:
+        chroma_client.delete_collection("documents")
+    except:
+        pass
+    
+    # Create a new collection
+    chroma_collection = chroma_client.get_or_create_collection("documents")
+    
+    # Create vector store
+    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    
+    # Load department-specific documents
+    documents = []
+    
+    # Add department-specific files
+    if dept_path.exists() and dept_path.is_dir():
+        for file_path in dept_path.glob("*"):
+            if file_path.is_file() and file_path.suffix in ['.md', '.txt', '.csv']:
+                print(f"Processing {file_path.name}...")
+                try:
+                    # Read the file content
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        content = f.read()
+                    
+                    # Create a document with metadata
+                    from llama_index.core import Document
+                    doc = Document(
+                        text=content,
+                        metadata={
+                            "source": str(file_path.name),
+                            "department": department,
+                            "type": "department_specific"
+                        }
+                    )
+                    documents.append(doc)
+                except Exception as e:
+                    print(f"Error processing {file_path}: {str(e)}")
+    
+    # Add general documents
+    if general_path.exists() and general_path.is_dir():
+        for file_path in general_path.glob("*"):
+            if file_path.is_file() and file_path.suffix in ['.md', '.txt', '.csv']:
+                print(f"Processing general document: {file_path.name}...")
+                try:
+                    # Read the file content
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        content = f.read()
+                    
+                    # Create a document with metadata
+                    from llama_index.core import Document
+                    doc = Document(
+                        text=content,
+                        metadata={
+                            "source": str(file_path.name),
+                            "department": "general",
+                            "type": "general"
+                        }
+                    )
+                    documents.append(doc)
+                except Exception as e:
+                    print(f"Error processing general document {file_path}: {str(e)}")
+    
+    if not documents:
+        print(f"No documents found for {department} department.")
+        return
+    
+    print(f"Indexing {len(documents)} documents...")
+    
+    # Create index with the documents
+    index = VectorStoreIndex.from_documents(
+        documents,
+        storage_context=storage_context,
+        show_progress=True,
+        embed_model=embed_model
+    )
+    
+    print(f"✅ Successfully indexed {len(documents)} documents for {department} department")
+    print(f"Index stored in: {persist_dir}")
+
+def main():
+    """Main function to process documents for all departments"""
+    departments = ["hr", "engineering", "finance", "marketing"]
+    
+    for dept in departments:
+        print(f"\n{'='*50}")
+        print(f"Processing {dept.upper()} department")
+        print(f"{'='*50}")
+        process_documents(dept)
+    
+    print("\n✅ Document processing completed for all departments!")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,18 @@
+streamlit
+llama-index
+chromadb
+cohere
+python-dotenv
+pandas
+python-multipart
+llama-index-core
+llama-index-readers-file
+llama-index-embeddings-cohere
+ipython
+llama-index-vector-stores-chroma
+llama-index-llms-groq
+pysqlite3-binary
+pymongo
+bcrypt
+openinference-instrumentation-llama-index 
+arize-otel

test_mongodb_integration.py

@@ -0,0 +1,129 @@
+"""
+Test script for MongoDB integration.
+Run this script to verify the MongoDB connection and user management.
+"""
+import os
+import sys
+from dotenv import load_dotenv
+from database import db, initialize_users
+
+# Load environment variables from .env file
+load_dotenv()
+
+def test_connection():
+    """Test MongoDB connection"""
+    try:
+        # Test the connection by pinging the database
+        db.client.admin.command('ping')
+        print("✅ Successfully connected to MongoDB")
+        return True
+    except Exception as e:
+        print(f"❌ Failed to connect to MongoDB: {e}")
+        return False
+
+def test_user_management():
+    """Test user management functions"""
+    test_username = "test_user_123"
+    test_password = "test_password_123"
+    test_role = "test_role"
+    
+    # Clean up test user if exists
+    db.users.delete_one({"username": test_username})
+    
+    # Test add_user
+    print("\nTesting add_user...")
+    if db.add_user(test_username, test_password, test_role):
+        print(f"✅ Successfully added test user: {test_username}")
+    else:
+        print("❌ Failed to add test user")
+        return False
+    
+    # Test verify_user with correct password
+    print("\nTesting verify_user with correct password...")
+    user = db.verify_user(test_username, test_password)
+    if user and user["username"] == test_username and user["role"] == test_role:
+        print("✅ Successfully verified user with correct password")
+    else:
+        print("❌ Failed to verify user with correct password")
+        return False
+    
+    # Test verify_user with incorrect password
+    print("\nTesting verify_user with incorrect password...")
+    user = db.verify_user(test_username, "wrong_password")
+    if user is None:
+        print("✅ Correctly rejected incorrect password")
+    else:
+        print("❌ Incorrectly accepted wrong password")
+        return False
+    
+    # Test get_user
+    print("\nTesting get_user...")
+    user = db.get_user(test_username)
+    if user and user["username"] == test_username and user["role"] == test_role:
+        print("✅ Successfully retrieved user details")
+    else:
+        print("❌ Failed to retrieve user details")
+        return False
+    
+    # Clean up
+    db.users.delete_one({"username": test_username})
+    return True
+
+def test_initialize_users():
+    """Test user initialization"""
+    print("\nTesting user initialization...")
+    try:
+        # Clean up any existing test users
+        test_usernames = ["Tony", "Bruce", "Sam", "Peter", "Sid", "Natasha"]
+        db.users.delete_many({"username": {"$in": test_usernames}})
+        
+        success_count, total_users, errors = initialize_users()
+        
+        if errors:
+            print(f"⚠️  Encountered {len(errors)} errors during user initialization:")
+            for error in errors:
+                print(f"   - {error}")
+        
+        if success_count == total_users:
+            print(f"✅ Successfully initialized {success_count}/{total_users} users")
+            return True
+        else:
+            print(f"❌ Only initialized {success_count}/{total_users} users")
+            return False
+    except Exception as e:
+        print(f"❌ Error during user initialization test: {str(e)}")
+        return False
+
+if __name__ == "__main__":
+    print("=== Testing MongoDB Integration ===")
+    
+    # Check if required environment variables are set
+    required_vars = ['MONGO_URI']
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    
+    if missing_vars:
+        print("❌ Missing required environment variables:")
+        for var in missing_vars:
+            print(f"   - {var}")
+        print("\nPlease create a .env file with these variables. See .env.example")
+        sys.exit(1)
+    
+    # Run tests
+    connection_ok = test_connection()
+    
+    if connection_ok:
+        print("\n=== Running User Management Tests ===")
+        user_tests_ok = test_user_management()
+        
+        print("\n=== Running User Initialization Test ===")
+        init_ok = test_initialize_users()
+        
+        if user_tests_ok and init_ok:
+            print("\n✅ All tests passed!")
+            sys.exit(0)
+        else:
+            print("\n❌ Some tests failed")
+            sys.exit(1)
+    else:
+        print("\n❌ Connection test failed. Please check your MongoDB connection details.")
+        sys.exit(1)