Initial commit with all files including LFS

.dockerignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.sqlite3
+.env
+.env.*
+.git/
+.gitignore
+.vscode/
+.idea/
+logs/
+data/new_data/
+**/__pycache__/
+**/*.py[cod]

.env.example

@@ -0,0 +1,8 @@
+# OpenAI API Configuration (Optional - only needed for AI chat feature)
+OPENAI_API_KEY=your_openai_api_key_here
+
+# LangSmith Configuration (Optional - for tracing)
+LANGSMITH_API_KEY=your_langsmith_api_key_here
+LANGSMITH_PROJECT=hbv-ai-assistant
+LANGCHAIN_PROJECT=hbv-ai-assistant
+LANGSMITH_URL=https://api.smith.langchain.com

.gitattributes

@@ -0,0 +1,4 @@
+*.faiss filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,212 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+*.ipynb
+*.docx
+*.pptx

API_DOCUMENTATION.md

@@ -0,0 +1,413 @@
+# HBV AI Assistant - API Documentation for Frontend Engineers
+
+## Table of Contents
+- [Overview](#overview)
+- [Base URL](#base-url)
+- [Core Endpoints](#core-endpoints)
+  - [1. POST /assess](#1-post-assess)
+  - [2. POST /assess/text](#2-post-assesstext)
+  - [3. POST /ask](#3-post-ask)
+  - [4. POST /ask/stream](#4-post-askstream)
+  - [5. GET /health](#5-get-health)
+- [Additional Endpoints](#additional-endpoints)
+- [Error Handling](#error-handling)
+- [Code Examples](#code-examples)
+
+---
+
+## Overview
+
+The HBV AI Assistant API provides endpoints for evaluating patient eligibility for HBV (Hepatitis B Virus) treatment according to SASLT 2021 guidelines. The API supports both structured data input and free-form text parsing, along with an interactive AI chat for guideline exploration.
+
+**API Version:** 1.0.0
+
+---
+
+## Base URL
+
+```
+Development: http://127.0.0.1:8000
+Production: [Your production URL]
+```
+
+---
+
+## Core Endpoints
+
+### 1. POST /assess
+
+**Primary endpoint for HBV patient eligibility assessment with structured data input.**
+
+#### Request
+
+**URL:** `/assess`  
+**Method:** `POST`  
+**Content-Type:** `application/json`
+
+**Request Body Schema:**
+
+```json
+{
+  "sex": "Male",                              // Required: "Male" or "Female"
+  "age": 45,                                  // Required: 0-120
+  "pregnancy_status": "Not pregnant",         // Required: "Not pregnant" or "Pregnant"
+  "hbsag_status": "Positive",                 // Required: "Positive" or "Negative"
+  "duration_hbsag_months": 12,                // Required: Duration in months (≥0)
+  "hbv_dna_level": 5000,                      // Required: HBV DNA in IU/mL (≥0)
+  "hbeag_status": "Positive",                 // Required: "Positive" or "Negative"
+  "alt_level": 80,                            // Required: ALT in U/L (≥0)
+  "fibrosis_stage": "F2-F3",                  // Required: "F0-F1", "F2-F3", or "F4"
+  "necroinflammatory_activity": "A2",         // Required: "A0", "A1", "A2", or "A3"
+  "extrahepatic_manifestations": false,       // Required: true or false
+  "immunosuppression_status": "None",         // Optional: "None", "Chemotherapy", "Other"
+  "coinfections": [],                         // Optional: Array of "HIV", "HCV", "HDV"
+  "family_history_cirrhosis_hcc": false,      // Required: true or false
+  "other_comorbidities": []                   // Optional: Array of strings
+}
+```
+
+#### Response
+
+**Status Code:** `200 OK`
+
+**Response Body:**
+
+```json
+{
+  "eligible": true,
+  "recommendations": "Based on SASLT 2021 guidelines, this patient IS ELIGIBLE for HBV antiviral treatment.\n\n**Eligibility Criteria Met:**\n- HBeAg-positive chronic hepatitis B with HBV DNA >2,000 IU/mL and ALT elevation (80 U/L) [SASLT 2021, Page 15]\n- Significant fibrosis (F2-F3) with moderate necroinflammatory activity (A2) [SASLT 2021, Page 18]\n\n**Treatment Recommendations:**\n1. **First-line therapy:** Entecavir (ETV) 0.5mg daily or Tenofovir Disoproxil Fumarate (TDF) 300mg daily [SASLT 2021, Page 22]\n2. **Alternative:** Tenofovir Alafenamide (TAF) 25mg daily for patients with renal concerns [SASLT 2021, Page 23]\n\n**Monitoring:**\n- HBV DNA every 3 months during first year\n- ALT and complete blood count every 3-6 months\n- HBeAg/anti-HBe every 6-12 months [SASLT 2021, Page 28]"
+}
+```
+
+#### Field Descriptions
+
+| Field | Type | Description | Validation |
+|-------|------|-------------|------------|
+| `sex` | string | Patient's biological sex | "Male" or "Female" |
+| `age` | integer | Patient's age in years | 0-120 |
+| `pregnancy_status` | string | Current pregnancy status | "Not pregnant" or "Pregnant" |
+| `hbsag_status` | string | Hepatitis B surface antigen status | "Positive" or "Negative" |
+| `duration_hbsag_months` | integer | How long HBsAg has been positive | ≥0 |
+| `hbv_dna_level` | float | HBV DNA viral load | ≥0 IU/mL |
+| `hbeag_status` | string | Hepatitis B e-antigen status | "Positive" or "Negative" |
+| `alt_level` | float | Alanine aminotransferase level | ≥0 U/L |
+| `fibrosis_stage` | string | Liver fibrosis/cirrhosis stage | "F0-F1", "F2-F3", or "F4" |
+| `necroinflammatory_activity` | string | Degree of liver inflammation | "A0", "A1", "A2", or "A3" |
+| `extrahepatic_manifestations` | boolean | Presence of HBV-related conditions outside liver | true/false |
+| `immunosuppression_status` | string | Current immunosuppression | "None", "Chemotherapy", "Other" |
+| `coinfections` | array | Other viral infections | ["HIV", "HCV", "HDV"] or [] |
+| `family_history_cirrhosis_hcc` | boolean | First-degree relative with cirrhosis/HCC | true/false |
+| `other_comorbidities` | array | Additional medical conditions | Array of strings or [] |
+
+---
+
+### 2. POST /assess/text
+
+**Text-based HBV patient eligibility assessment - parses free-form clinical text.**
+
+#### Request
+
+**URL:** `/assess/text`  
+**Method:** `POST`  
+**Content-Type:** `application/json`
+
+**Request Body Schema:**
+
+```json
+{
+  "text_input": "45-year-old male patient\nHBsAg: Positive for 12 months\nHBV DNA: 5000 IU/mL\nHBeAg: Positive\nALT: 80 U/L\nFibrosis stage: F2-F3\nNecroinflammatory activity: A2\nNo extrahepatic manifestations\nNo immunosuppression\nNo coinfections (HIV, HCV, HDV)\nNo family history of cirrhosis or HCC"
+}
+```
+
+**Field:**
+- `text_input` (string, required): Free-form text containing patient clinical data. Minimum 10 characters.
+
+#### Response
+
+**Status Code:** `200 OK`
+
+**Response Body:** Same as `/assess` endpoint
+
+```json
+{
+  "eligible": true,
+  "recommendations": "Based on SASLT 2021 guidelines, this patient IS ELIGIBLE for HBV antiviral treatment..."
+}
+```
+
+#### How It Works
+
+1. **LLM-based parsing:** The API uses an AI model to extract structured patient data from the free-form text
+2. **Validation:** Extracted data is validated against the same schema as `/assess`
+3. **Assessment:** Performs identical assessment logic as the structured endpoint
+4. **Response:** Returns the same structured response format
+
+#### Text Input Tips
+
+- Include all relevant clinical parameters
+- Use clear labels (e.g., "HBV DNA:", "ALT:", "Age:")
+- Can use natural language or structured format
+- The AI will extract and interpret the data intelligently
+
+---
+
+### 3. POST /ask
+
+**Interactive AI chat for exploring HBV treatment guidelines.**
+
+#### Request
+
+**URL:** `/ask`  
+**Method:** `POST`  
+**Content-Type:** `application/json`
+
+**Request Body Schema:**
+
+```json
+{
+  "query": "What are the first-line treatment options for HBeAg-positive chronic hepatitis B?",
+  "session_id": "doctor_123_session_1",
+  "patient_context": {
+    // Optional: Include HBVPatientInput object for context
+    "age": 45,
+    "sex": "Male",
+    "hbv_dna_level": 5000,
+    // ... other patient fields
+  },
+  "assessment_result": {
+    // Optional: Include prior assessment result for context
+    "eligible": true,
+    "recommendations": "..."
+  }
+}
+```
+
+**Fields:**
+- `query` (string, required): Doctor's question about HBV guidelines. Max 2000 characters.
+- `session_id` (string, optional): Session identifier for conversation continuity. Default: "default"
+- `patient_context` (object, optional): Patient data from a prior assessment to provide context
+- `assessment_result` (object, optional): Assessment result to provide context
+
+#### Response
+
+**Status Code:** `200 OK`
+
+**Response Body:**
+
+```json
+{
+  "response": "According to SASLT 2021 guidelines, the first-line treatment options for HBeAg-positive chronic hepatitis B are:\n\n1. **Entecavir (ETV)** - 0.5mg once daily [SASLT 2021, Page 22]\n2. **Tenofovir Disoproxil Fumarate (TDF)** - 300mg once daily [SASLT 2021, Page 22]\n\nBoth medications have:\n- High barrier to resistance\n- Excellent efficacy in viral suppression\n- Well-established safety profiles\n\n**Alternative option:**\n- **Tenofovir Alafenamide (TAF)** - 25mg once daily, particularly for patients with renal concerns or bone disease [SASLT 2021, Page 23]",
+  "session_id": "doctor_123_session_1"
+}
+```
+
+#### Session Management
+
+- **Conversation continuity:** Use the same `session_id` to maintain conversation context across multiple questions
+- **New conversation:** Use a different `session_id` or omit it for a fresh conversation
+- **Clear session:** Use `DELETE /session/{session_id}` to clear conversation history
+
+---
+
+### 4. POST /ask/stream
+
+**Streaming version of the AI chat endpoint for real-time responses.**
+
+#### Request
+
+**URL:** `/ask/stream`  
+**Method:** `POST`  
+**Content-Type:** `application/json`
+
+**Request Body:** Same as `/ask` endpoint
+
+```json
+{
+  "query": "What monitoring is required during HBV treatment?",
+  "session_id": "doctor_123_session_1"
+}
+```
+
+#### Response
+
+**Status Code:** `200 OK`  
+**Content-Type:** `text/markdown`  
+**Transfer-Encoding:** `chunked`
+
+**Response:** Streaming text/markdown chunks
+
+The response streams in real-time as the AI generates the answer. This provides a better user experience for longer responses.
+
+
+---
+
+### 5. GET /health
+
+**Simple health check endpoint to verify API is running.**
+
+#### Request
+
+**URL:** `/health`  
+**Method:** `GET`
+
+#### Response
+
+**Status Code:** `200 OK`
+
+**Response Body:**
+
+```json
+{
+  "status": "healthy",
+  "version": "1.0.0",
+  "timestamp": "2024-01-15T10:30:00.123456"
+}
+```
+
+**Fields:**
+- `status` (string): API health status ("healthy")
+- `version` (string): API version
+- `timestamp` (string): Current server timestamp in ISO format
+
+#### Use Case
+
+Use this endpoint to:
+- Verify the API is running and accessible
+- Monitor API availability
+- Check server time for debugging
+
+---
+
+## Additional Endpoints
+
+### API Information
+
+**GET** `/`
+
+**Response:**
+```json
+{
+  "name": "HBV AI Assistant API",
+  "version": "1.0.0",
+  "description": "HBV Patient Selection System - Evaluates patient eligibility for HBV treatment according to SASLT 2021 guidelines",
+  "docs": "/docs",
+  "endpoints": {
+    "assess": "/assess (POST) - Primary endpoint for HBV patient eligibility assessment",
+    "assess_text": "/assess/text (POST) - Text-based HBV patient eligibility assessment",
+    "ask": "/ask (POST) - Optional AI chat for guideline exploration",
+    "ask_stream": "/ask/stream (POST) - Streaming AI chat responses",
+    "health": "/health (GET) - Simple health check"
+  }
+}
+```
+
+---
+
+## Error Handling
+
+### HTTP Status Codes
+
+| Code | Description |
+|------|-------------|
+| 200 | Success |
+| 400 | Bad Request - Invalid input data |
+| 422 | Validation Error - Request body doesn't match schema |
+| 429 | Too Many Requests - Rate limit exceeded |
+| 500 | Internal Server Error |
+
+### Error Response Format
+
+```json
+{
+  "detail": "Error message describing what went wrong"
+}
+```
+
+### Validation Error Example
+
+**Request with invalid data:**
+```json
+{
+  "sex": "Unknown",  // Invalid: must be "Male" or "Female"
+  "age": 150         // Invalid: must be 0-120
+}
+```
+
+**Response (422):**
+```json
+{
+  "detail": [
+    {
+      "loc": ["body", "sex"],
+      "msg": "Sex must be either Male or Female",
+      "type": "value_error"
+    },
+    {
+      "loc": ["body", "age"],
+      "msg": "ensure this value is less than or equal to 120",
+      "type": "value_error.number.not_le"
+    }
+  ]
+}
+```
+---
+
+## Interactive API Documentation
+
+For interactive API documentation with a built-in testing interface, visit:
+
+**Swagger UI:** `http://127.0.0.1:8000/docs`  
+**ReDoc:** `http://127.0.0.1:8000/redoc`
+
+These interfaces allow you to:
+- View all endpoints and their schemas
+- Test endpoints directly from the browser
+- See example requests and responses
+- Understand validation rules
+
+---
+
+## Rate Limiting
+
+The API implements rate limiting:
+- **Default limit:** 100 requests per minute per IP address
+- **Response header:** `X-RateLimit-Remaining` shows remaining requests
+- **Status code:** 429 when limit exceeded
+
+---
+
+## Best Practices
+
+1. **Error Handling:**
+   - Always check `response.ok` before parsing JSON
+   - Display validation errors to users clearly
+   - Implement retry logic for 500 errors
+
+2. **Session Management:**
+   - Use unique session IDs for different patient conversations
+   - Clear sessions when switching patients
+   - Consider using user ID + timestamp for session IDs
+
+3. **Performance:**
+   - Use `/ask/stream` for better UX on longer responses
+   - Cache assessment results when appropriate
+   - Consider debouncing text input for `/assess/text` endpoint
+
+4. **Data Validation:**
+   - Validate input on frontend before sending to API
+   - Use the exact enum values specified in the documentation
+   - Handle validation errors gracefully
+
+---
+
+## Support
+
+For questions or issues:
+- Check the interactive documentation at `/docs`
+- Review error messages carefully - they indicate what's wrong
+- Ensure all required fields are provided with correct data types
+
+---
+
+**Last Updated:** 2024-01-15  
+**API Version:** 1.0.0

Dockerfile

@@ -0,0 +1,50 @@
+# ========================
+# Stage 1 - Builder
+# ========================
+FROM python:3.11-slim AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy only requirements first (better caching)
+COPY requirements.txt .
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# ========================
+# Stage 2 - Final Runtime Image
+# ========================
+FROM python:3.11-slim
+
+# Install minimal runtime dependencies
+RUN apt-get update && apt-get install -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user
+RUN useradd --create-home --shell /bin/bash appuser
+WORKDIR /app
+
+# Copy installed packages and application code
+COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+COPY . /app
+
+# Set permissions
+RUN chown -R appuser:appuser /app
+USER appuser
+
+# Expose port
+EXPOSE 7860
+
+# Set the working directory to the backend folder
+WORKDIR /app
+
+# Command to run the FastAPI app
+CMD ["sh", "-c", "uvicorn api.app:app --host 0.0.0.0 --port 7860"]

README.md

@@ -0,0 +1,237 @@
+---
+title: HBV AI Assistant - Patient Selection System
+emoji: 🏥
+colorFrom: blue
+colorTo: green
+sdk: docker
+pinned: false
+app_port: 7860
+---
+
+# HBV AI Assistant - Patient Selection System
+
+A specialized AI-powered clinical decision support system for hepatologists and healthcare professionals managing Hepatitis B Virus (HBV) patients. The system evaluates patient eligibility for treatment according to SASLT 2021 guidelines and provides evidence-based recommendations.
+
+## 🎯 Features
+
+### Core Capabilities
+- **Patient Eligibility Assessment**: Evaluates HBV patients for treatment eligibility based on SASLT 2021 guidelines
+- **Evidence-Based Guidance**: Provides treatment recommendations according to authoritative medical guidelines
+- **Comprehensive Input Validation**: Validates patient data including HBV DNA levels, ALT levels, fibrosis stage, and more
+- **Treatment Options**: Recommends preferred regimens (ETV, TDF, TAF) based on patient profile
+- **AI-Powered Chat**: Interactive AI bot for exploring guideline recommendations
+- **JSON API**: RESTful POST endpoint for programmatic integration
+
+### Technical Features
+- **FastAPI Backend**: High-performance async API
+- **Structured Input/Output**: Well-defined schemas for patient data and eligibility results
+- **Real-time Processing**: Fast eligibility determination
+- **Authentication**: Secure session-based authentication
+- **Rate Limiting**: Built-in API rate limiting
+- **CORS Support**: Cross-origin resource sharing enabled
+
+## 🚀 Deployment
+
+### Live API
+The API is deployed at: **http://127.0.0.1:7860**
+
+### Quick Start
+
+1. **Access the API**:
+   - API Docs: http://127.0.0.1:7860/docs
+   - Health Check: http://127.0.0.1:7860/health
+
+2. **Submit Patient Data**:
+   - Use the POST `/assess` endpoint to evaluate patient eligibility
+   - Provide patient information according to the input schema
+   - Receive eligibility determination and treatment recommendations
+
+### Deploy Your Own Instance
+
+See [DEPLOYMENT.md](DEPLOYMENT.md) for detailed deployment instructions.
+
+## 📚 API Endpoints
+
+### Health & Status
+- `GET /` - API information
+- `GET /health` - Health check
+
+### HBV Patient Assessment
+- `POST /assess` - Evaluate patient eligibility for HBV treatment using structured data
+- `POST /assess/text` - Text-based patient assessment (provide clinical notes in free text format)
+  - **Input**: Patient data (sex, age, HBV DNA, ALT, fibrosis stage, etc.)
+  - **Output**: Eligibility status and treatment recommendations
+
+### AI Chat 
+- `POST /ask` - Ask guideline-related questions with optional patient context
+  - **Input**: 
+    - `query` (string): The question or message
+    - `session_id` (string, optional): Session identifier for conversation history
+    - `patient_context` (object, optional): Patient data for context-aware responses
+    - `assessment_result` (object, optional): Previous assessment results for reference
+  
+- `POST /ask/stream` - Streaming chat responses with optional patient context
+  - **Input**: Same as `/ask` endpoint
+  - **Output**: Stream of text chunks for real-time display
+
+## 💻 Local Development
+
+### Prerequisites
+- Python 3.11+
+- OpenAI API key (optional, for AI chat feature)
+
+### Setup
+
+1. **Clone the repository**:
+```bash
+git clone https://github.com/your-repo/hbv-ai-assistant.git
+cd hbv-ai-assistant
+```
+
+2. **Install dependencies**:
+```bash
+pip install -r requirements.txt
+```
+
+3. **Configure environment variables**:
+```bash
+cp .env.example .env
+# Edit .env with your API keys
+```
+
+4. **Run the application**:
+```bash
+python app.py
+```
+
+5. **Access the application**:
+   - API: http://localhost:7860
+   - Docs: http://localhost:7860/docs
+   - Test the `/assess` endpoint with patient data
+
+## 🔧 Configuration
+
+### Environment Variables
+
+See `.env.example` for all configuration options:
+
+- `OPENAI_API_KEY`: Your OpenAI API key (optional, for AI chat)
+- `PORT`: Server port (default: 7860)
+- `ALLOWED_ORIGINS`: CORS allowed origins
+
+### Authentication
+
+Default credentials (change in production):
+- Username: `admin`
+- Password: `admin123`
+
+Update in `api/routers/auth.py` or via environment variables.
+
+## 📖 Usage Examples
+
+### Assessing Patient Eligibility
+
+```python
+import requests
+
+# Login
+response = requests.post(
+    "http://127.0.0.1:7860/auth/login",
+    json={"username": "admin", "password": "admin123"}
+)
+cookies = response.cookies
+
+# Assess patient eligibility
+patient_data = {
+    "sex": "Male",
+    "age": 45,
+    "pregnancy_status": "Not pregnant",
+    "hbsag_status": "Positive",
+    "duration_hbsag_months": 12,
+    "hbv_dna_level": 50000,
+    "hbeag_status": "Positive",
+    "alt_level": 60,
+    "fibrosis_stage": "F2-F3",
+    "necroinflammatory_activity": "A2",
+    "extrahepatic_manifestations": False,
+    "coinfections": [],
+    "family_history_cirrhosis_hcc": False
+}
+
+response = requests.post(
+    "http://127.0.0.1:7860/assess",
+    json=patient_data,
+    cookies=cookies
+)
+result = response.json()
+print(f"Eligible: {result['eligible']}")
+print(f"Recommendations: {result['recommendations']}")
+```
+
+## 🏗️ Architecture
+
+### Components
+
+- **FastAPI Backend**: RESTful API with async support
+- **Eligibility Engine**: Evaluates patient data against SASLT 2021 criteria
+- **AI Chat (Optional)**: LangChain-powered conversational interface for guideline exploration
+- **Validation Layer**: Ensures data integrity and completeness
+
+### Assessment Logic
+
+The system evaluates patients based on SASLT 2021 criteria:
+
+1. **HBV DNA > 2,000 IU/mL** + **ALT > ULN** + moderate necroinflammation/fibrosis (≥F2 or ≥A2)
+2. **Cirrhosis** (F4) with any detectable HBV DNA
+3. **HBV DNA > 20,000 IU/mL** + **ALT > 2×ULN** regardless of fibrosis
+4. **Age > 30** with HBeAg-positive chronic infection (normal ALT, high HBV DNA)
+5. **Family history** of HCC/cirrhosis + HBV DNA > 2,000 + ALT > ULN
+6. **Extrahepatic manifestations**
+
+## 📊 Response Format
+
+The API returns:
+- **Eligibility Status**: Eligible / Not Eligible
+- **Guideline Recommendations**: Specific criteria met and treatment options
+- **Treatment Choices**: Preferred regimens (ETV, TDF, TAF)
+
+Example Response:
+```json
+{
+  "eligible": true,
+  "recommendations": "Patient meets SASLT 2021 criteria: HBV DNA > 2,000 IU/mL, ALT > ULN, and fibrosis stage F2-F3 (Grade A)",
+  "treatment_options": ["ETV", "TDF", "TAF"],
+  "guideline": "SASLT 2021"
+}
+```
+
+## 🔒 Security
+
+- Session-based authentication
+- Rate limiting (100 requests/minute)
+- CORS protection
+- Input validation
+- Secure cookie handling
+
+## 📝 License
+
+[Add your license here]
+
+## 🤝 Contributing
+
+Contributions are welcome! Please read the contributing guidelines first.
+
+## 📧 Support
+
+For issues or questions:
+- Check the [DEPLOYMENT.md](DEPLOYMENT.md) guide
+- Review API docs at `/docs`
+- Open an issue on GitHub
+
+## 🙏 Acknowledgments
+
+Built with:
+- FastAPI
+- Pydantic
+- Python 3.11+
+- SASLT 2021 Guidelines

api/app.py

@@ -0,0 +1,118 @@
+
+import logging
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, HTTPException
+from fastapi.exceptions import RequestValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+# Import routers
+from api.routers import medical, hbv_assessment
+from api.middleware import (
+    ProcessTimeMiddleware, 
+    LoggingMiddleware, 
+    RateLimitMiddleware,
+    get_cors_middleware_config
+)
+from fastapi.middleware.cors import CORSMiddleware
+from api.exceptions import (
+    http_exception_handler,
+    validation_exception_handler,
+    general_exception_handler,
+    starlette_exception_handler
+)
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan management with background initialization"""
+    # Startup
+    logger.info("Starting HBV AI Assistant API...")
+    
+    # Start background initialization of heavy components (optional for AI chat)
+    try:
+        from core.background_init import start_background_initialization
+        logger.info("🚀 Starting background initialization of components...")
+        start_background_initialization()
+        logger.info("API started successfully (components loading in background)")
+    except Exception as e:
+        logger.error(f"Failed to start background initialization: {e}")
+        logger.info("API started with lazy loading fallback")
+    
+    yield
+    
+    # Shutdown
+    logger.info("Shutting down HBV AI Assistant API...")
+
+
+# Create FastAPI application
+app = FastAPI(
+    title="HBV AI Assistant API",
+    description="HBV Patient Selection System - Evaluates patient eligibility for HBV treatment according to SASLT 2021 guidelines",
+    version="1.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+    lifespan=lifespan
+)
+
+# Add middleware
+app.add_middleware(CORSMiddleware, **get_cors_middleware_config())
+app.add_middleware(ProcessTimeMiddleware)
+app.add_middleware(LoggingMiddleware)
+app.add_middleware(RateLimitMiddleware, calls_per_minute=100)  # Adjust as needed
+
+# Add exception handlers
+app.add_exception_handler(HTTPException, http_exception_handler)
+app.add_exception_handler(RequestValidationError, validation_exception_handler)
+app.add_exception_handler(StarletteHTTPException, starlette_exception_handler)
+app.add_exception_handler(Exception, general_exception_handler)
+
+# Include routers
+app.include_router(hbv_assessment.router)  # Primary HBV assessment endpoint
+app.include_router(medical.router)  # Optional AI chat for guideline exploration
+
+# Root endpoint
+@app.get("/")
+async def root():
+    """Root endpoint with API information"""
+    return {
+        "name": "HBV AI Assistant API",
+        "version": "1.0.0",
+        "description": "HBV Patient Selection System - Evaluates patient eligibility for HBV treatment according to SASLT 2021 guidelines",
+        "docs": "/docs",
+        "endpoints": {
+            "assess": "/assess (POST) - Primary endpoint for HBV patient eligibility assessment",
+            "assess_text": "/assess/text (POST) - Text-based HBV patient eligibility assessment",
+            "ask": "/ask (POST) - Optional AI chat for guideline exploration",
+            "ask_stream": "/ask/stream (POST) - Streaming AI chat responses",
+            "health": "/health (GET) - Simple health check",
+        }
+    }
+
+
+# Simple health check endpoint
+@app.get("/health")
+async def health_check():
+    """Simple health check endpoint"""
+    from datetime import datetime
+    return {
+        "status": "healthy",
+        "version": "1.0.0",
+        "timestamp": datetime.now().isoformat()
+    }
+
+
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "api.app:app",
+        host="127.0.0.1",
+        port=8000,
+        reload=True,
+        log_level="info"
+    )

api/exceptions.py

@@ -0,0 +1,86 @@
+"""
+Exception handlers for Medical RAG AI Advisor API
+"""
+import logging
+from datetime import datetime
+from fastapi import Request, HTTPException
+from fastapi.responses import JSONResponse
+from fastapi.exceptions import RequestValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from api.models import ErrorResponse
+
+logger = logging.getLogger(__name__)
+
+
+async def http_exception_handler(request: Request, exc: HTTPException):
+    """Handle HTTP exceptions"""
+    logger.error(f"HTTP Exception: {exc.status_code} - {exc.detail}")
+    
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=ErrorResponse(
+            error="HTTP_ERROR",
+            message=exc.detail,
+            timestamp=datetime.now().isoformat()
+        ).dict()
+    )
+
+
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    """Handle request validation errors"""
+    errors = exc.errors()
+    
+    # Convert errors to JSON-serializable format
+    serializable_errors = []
+    for error in errors:
+        error_dict = {
+            "type": error.get("type"),
+            "loc": list(error.get("loc", [])),
+            "msg": error.get("msg"),
+            "input": str(error.get("input"))  # Convert to string to ensure serializability
+        }
+        if "ctx" in error:
+            error_dict["ctx"] = {k: str(v) for k, v in error["ctx"].items()}
+        serializable_errors.append(error_dict)
+    
+    logger.error(f"Validation Error: {serializable_errors}")
+    
+    return JSONResponse(
+        status_code=422,
+        content=ErrorResponse(
+            error="VALIDATION_ERROR",
+            message="Request validation failed",
+            details={"validation_errors": serializable_errors},
+            timestamp=datetime.now().isoformat()
+        ).dict()
+    )
+
+
+async def general_exception_handler(request: Request, exc: Exception):
+    """Handle general exceptions"""
+    logger.error(f"Unhandled Exception: {type(exc).__name__} - {str(exc)}")
+    
+    return JSONResponse(
+        status_code=500,
+        content=ErrorResponse(
+            error="INTERNAL_SERVER_ERROR",
+            message="An internal server error occurred",
+            details={"exception_type": type(exc).__name__},
+            timestamp=datetime.now().isoformat()
+        ).dict()
+    )
+
+
+async def starlette_exception_handler(request: Request, exc: StarletteHTTPException):
+    """Handle Starlette HTTP exceptions"""
+    logger.error(f"Starlette HTTP Exception: {exc.status_code} - {exc.detail}")
+    
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=ErrorResponse(
+            error="HTTP_ERROR",
+            message=exc.detail,
+            timestamp=datetime.now().isoformat()
+        ).dict()
+    )

api/middleware.py

@@ -0,0 +1,177 @@
+"""
+Middleware for Medical RAG AI Advisor API
+"""
+import time
+import logging
+from typing import Callable, Awaitable, Optional
+from fastapi import Request, Response, HTTPException, Cookie
+from fastapi.middleware.cors import CORSMiddleware
+from starlette.middleware.base import BaseHTTPMiddleware
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessTimeMiddleware(BaseHTTPMiddleware):
+    """Middleware to add processing time to response headers"""
+    
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        start_time = time.time()
+        response = await call_next(request)
+        process_time = time.time() - start_time
+        response.headers["X-Process-Time"] = f"{process_time:.4f}"
+        return response
+
+
+class LoggingMiddleware(BaseHTTPMiddleware):
+    """Middleware for request/response logging"""
+    
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        start_time = time.time()
+        
+        # Log request
+        logger.info(f"Request: {request.method} {request.url}")
+        
+        try:
+            response = await call_next(request)
+            process_time = time.time() - start_time
+            
+            # Log response
+            logger.info(
+                f"Response: {response.status_code} - "
+                f"Time: {process_time:.4f}s - "
+                f"Path: {request.url.path}"
+            )
+            
+            return response
+            
+        except Exception as e:
+            process_time = time.time() - start_time
+            logger.error(
+                f"Error: {str(e)} - "
+                f"Time: {process_time:.4f}s - "
+                f"Path: {request.url.path}"
+            )
+            raise
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Simple rate limiting middleware"""
+    
+    def __init__(self, app, calls_per_minute: int = 60):
+        super().__init__(app)
+        self.calls_per_minute = calls_per_minute
+        self.client_calls = {}
+    
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        client_ip = request.client.host
+        current_time = time.time()
+        
+        # Clean old entries
+        self.client_calls = {
+            ip: calls for ip, calls in self.client_calls.items()
+            if any(call_time > current_time - 60 for call_time in calls)
+        }
+        
+        # Check rate limit
+        if client_ip in self.client_calls:
+            recent_calls = [
+                call_time for call_time in self.client_calls[client_ip]
+                if call_time > current_time - 60
+            ]
+            if len(recent_calls) >= self.calls_per_minute:
+                raise HTTPException(
+                    status_code=429,
+                    detail="Rate limit exceeded. Please try again later."
+                )
+            self.client_calls[client_ip] = recent_calls + [current_time]
+        else:
+            self.client_calls[client_ip] = [current_time]
+        
+        return await call_next(request)
+
+
+class AuthenticationMiddleware(BaseHTTPMiddleware):
+    """Middleware to protect endpoints with session authentication"""
+    
+    # Paths that don't require authentication
+    PUBLIC_PATHS = [
+        "/",
+        "/docs",
+        "/redoc",
+        "/openapi.json",
+        "/health",
+        "/auth/login",
+        "/auth/status",
+        "/assess",  # Allow assess endpoint for local testing
+        "/ask",     # Allow ask endpoint for local testing
+    ]
+    
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        # For local testing, disable authentication
+        # TODO: Enable authentication in production
+        return await call_next(request)
+        
+        # Original authentication code (disabled for local testing)
+        # # Check if path is public
+        # path = request.url.path
+        # 
+        # # Allow public paths
+        # if any(path.startswith(public_path) for public_path in self.PUBLIC_PATHS):
+        #     return await call_next(request)
+        # 
+        # # Check for session token
+        # session_token = request.cookies.get("session_token")
+        # 
+        # if not session_token:
+        #     raise HTTPException(
+        #         status_code=401,
+        #         detail="Authentication required"
+        #     )
+        # 
+        # # Verify session
+        # from api.routers.auth import verify_session
+        # session_data = verify_session(session_token)
+        # 
+        # if not session_data:
+        #     raise HTTPException(
+        #         status_code=401,
+        #         detail="Invalid or expired session"
+        #     )
+        # 
+        # # Add user info to request state
+        # request.state.user = session_data.get("username")
+        # 
+        # return await call_next(request)
+
+
+def get_cors_middleware_config():
+    """Get CORS middleware configuration"""
+    import os
+    
+    # Get allowed origins from environment or use defaults
+    allowed_origins = os.getenv("ALLOWED_ORIGINS", "").split(",")
+    if not allowed_origins or allowed_origins == [""]:
+        # Default to allowing Hugging Face Space and localhost
+        # Include null for file:// protocol and common local development origins
+        allowed_origins = [
+            "http://127.0.0.1:7860",
+            "https://huggingface.co",
+            "http://localhost:8000",
+            "http://127.0.0.1:8000",
+            "http://localhost:8080",  # Frontend server
+            "http://127.0.0.1:8080",
+            "http://localhost:5500",  # Live Server default port
+            "http://127.0.0.1:5500",
+            "http://localhost:3000",  # Common dev server port
+            "http://127.0.0.1:3000",
+            "null",  # For file:// protocol
+            "*"  # Allow all origins for local testing
+        ]
+    
+    return {
+        "allow_origins": ["*"],  # Allow all origins for local testing
+        "allow_credentials": False,  # Must be False when allow_origins is "*"
+        "allow_methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+        "allow_headers": ["*"],  # Allow all headers
+        "expose_headers": ["Set-Cookie"],
+    }

api/models.py

@@ -0,0 +1,188 @@
+"""
+API Models and Schemas for HBV AI Assistant
+"""
+from pydantic import BaseModel, Field, validator
+from typing import Optional, List, Dict, Any
+from enum import Enum
+
+
+class QueryType(str, Enum):
+    """Types of medical queries supported"""
+    GENERAL = "general"
+    DRUG_INTERACTION = "drug_interaction"
+    SIDE_EFFECTS = "side_effects"
+    GUIDELINES = "guidelines"
+    COMPARISON = "comparison"
+
+
+class QueryRequest(BaseModel):
+    """Request model for medical queries"""
+    query: str = Field(..., description="Medical question or query", min_length=1)
+    query_type: Optional[QueryType] = Field(None, description="Type of medical query")
+    context: Optional[str] = Field(None, description="Additional context for the query")
+    patient_info: Optional[Dict[str, Any]] = Field(None, description="Patient information if relevant")
+
+
+class QueryResponse(BaseModel):
+    """Response model for medical queries"""
+    response: str = Field(..., description="AI-generated medical response")
+    sources: Optional[List[str]] = Field(None, description="Sources used for the response")
+    confidence: Optional[float] = Field(None, description="Confidence score of the response")
+    query_type: Optional[QueryType] = Field(None, description="Detected or specified query type")
+    processing_time: Optional[float] = Field(None, description="Time taken to process the query")
+
+
+class StreamChunk(BaseModel):
+    """Model for streaming response chunks"""
+    chunk: str = Field(..., description="Chunk of the streaming response")
+    is_final: bool = Field(False, description="Whether this is the final chunk")
+
+
+class SideEffectReport(BaseModel):
+    """Model for side effect reporting"""
+    drug_name: str = Field(..., description="Name of the drug")
+    side_effects: str = Field(..., description="Reported side effects")
+    patient_age: Optional[int] = Field(None, description="Patient age")
+    patient_gender: Optional[str] = Field(None, description="Patient gender")
+    dosage: Optional[str] = Field(None, description="Drug dosage")
+    duration: Optional[str] = Field(None, description="Duration of treatment")
+    severity: Optional[str] = Field(None, description="Severity of side effects")
+    outcome: Optional[str] = Field(None, description="Outcome of the side effects")
+    additional_details: Optional[str] = Field(None, description="Additional clinical details")
+
+
+class SideEffectResponse(BaseModel):
+    """Response model for side effect reporting"""
+    report_id: str = Field(..., description="Unique identifier for the report")
+    status: str = Field(..., description="Status of the report submission")
+    message: str = Field(..., description="Confirmation message")
+    recommendations: Optional[List[str]] = Field(None, description="Clinical recommendations")
+
+
+class ComparisonRequest(BaseModel):
+    """Request model for provider/treatment comparisons"""
+    providers: List[str] = Field(..., description="List of providers to compare", min_items=2)
+    criteria: Optional[List[str]] = Field(None, description="Specific criteria for comparison")
+
+
+class ComparisonResponse(BaseModel):
+    """Response model for provider/treatment comparisons"""
+    comparison: str = Field(..., description="Detailed comparison analysis")
+    summary: Dict[str, Any] = Field(..., description="Summary of key differences")
+    recommendations: Optional[str] = Field(None, description="Recommendations based on comparison")
+
+
+class InitializationStatus(BaseModel):
+    """Initialization status response model"""
+    is_complete: bool = Field(..., description="Whether initialization is complete")
+    status_message: str = Field(..., description="Current initialization status")
+    is_successful: bool = Field(..., description="Whether initialization was successful")
+    error: Optional[str] = Field(None, description="Initialization error if any")
+
+
+class HealthStatus(BaseModel):
+    """Health check response model"""
+    status: str = Field(..., description="API health status")
+    version: str = Field(..., description="API version")
+    timestamp: str = Field(..., description="Current timestamp")
+    components: Dict[str, str] = Field(..., description="Status of system components")
+    initialization: Optional[InitializationStatus] = Field(None, description="Background initialization status")
+
+
+class ErrorResponse(BaseModel):
+    """Error response model"""
+    error: str = Field(..., description="Error type")
+    message: str = Field(..., description="Error message")
+    details: Optional[Dict[str, Any]] = Field(None, description="Additional error details")
+    timestamp: str = Field(..., description="Error timestamp")
+
+
+# ============================================================================
+# HBV PATIENT ASSESSMENT MODELS
+# ============================================================================
+
+class HBVPatientInput(BaseModel):
+    """Input model for HBV patient assessment"""
+    sex: str = Field(..., description="Patient sex: Male / Female")
+    age: int = Field(..., description="Patient age in years", ge=0, le=120)
+    pregnancy_status: str = Field(..., description="Pregnancy status: Not pregnant / Pregnant")
+    hbsag_status: str = Field(..., description="HBsAg status: Positive / Negative")
+    duration_hbsag_months: int = Field(..., description="Duration of HBsAg positivity in months", ge=0)
+    hbv_dna_level: float = Field(..., description="HBV DNA level in IU/mL", ge=0)
+    hbeag_status: str = Field(..., description="HBeAg status: Positive / Negative")
+    alt_level: float = Field(..., description="ALT level in U/L", ge=0)
+    fibrosis_stage: str = Field(..., description="Fibrosis/Cirrhosis stage: F0-F1 / F2-F3 / F4")
+    necroinflammatory_activity: str = Field(..., description="Necroinflammatory activity: A0 / A1 / A2 / A3")
+    extrahepatic_manifestations: bool = Field(..., description="Presence of extrahepatic manifestations")
+    immunosuppression_status: Optional[str] = Field(None, description="Immunosuppression status: None / Chemotherapy / Other")
+    coinfections: List[str] = Field(default_factory=list, description="Coinfections: HIV, HCV, HDV")
+    family_history_cirrhosis_hcc: bool = Field(..., description="Family history of Cirrhosis or HCC (first-degree relative)")
+    other_comorbidities: Optional[List[str]] = Field(None, description="Other comorbidities")
+    
+    @validator('sex')
+    def validate_sex(cls, v):
+        if v not in ['Male', 'Female']:
+            raise ValueError('Sex must be either Male or Female')
+        return v
+    
+    @validator('pregnancy_status')
+    def validate_pregnancy(cls, v):
+        if v not in ['Not pregnant', 'Pregnant']:
+            raise ValueError('Pregnancy status must be either "Not pregnant" or "Pregnant"')
+        return v
+    
+    @validator('hbsag_status')
+    def validate_hbsag(cls, v):
+        if v not in ['Positive', 'Negative']:
+            raise ValueError('HBsAg status must be either Positive or Negative')
+        return v
+    
+    @validator('hbeag_status')
+    def validate_hbeag(cls, v):
+        if v not in ['Positive', 'Negative']:
+            raise ValueError('HBeAg status must be either Positive or Negative')
+        return v
+    
+    @validator('fibrosis_stage')
+    def validate_fibrosis(cls, v):
+        if v not in ['F0-F1', 'F2-F3', 'F4']:
+            raise ValueError('Fibrosis stage must be F0-F1, F2-F3, or F4')
+        return v
+    
+    @validator('necroinflammatory_activity')
+    def validate_necroinflammatory(cls, v):
+        if v not in ['A0', 'A1', 'A2', 'A3']:
+            raise ValueError('Necroinflammatory activity must be A0, A1, A2, or A3')
+        return v
+
+
+class HBVAssessmentResponse(BaseModel):
+    """Response model for HBV patient assessment"""
+    eligible: bool = Field(..., description="Whether patient is eligible for treatment")
+    recommendations: str = Field(..., description="Detailed recommendations with citations from SASLT 2021 guidelines including page numbers")
+
+
+class TextAssessmentInput(BaseModel):
+    """Input model for text-based HBV patient assessment"""
+    text_input: str = Field(..., description="Free-form text containing patient data", min_length=10)
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "text_input": "45-year-old male patient\\nHBsAg: Positive for 12 months\\nHBV DNA: 5000 IU/mL\\nHBeAg: Positive\\nALT: 80 U/L\\nFibrosis stage: F2-F3\\nNecroinflammatory activity: A2\\nNo extrahepatic manifestations\\nNo immunosuppression\\nNo coinfections (HIV, HCV, HDV)\\nNo family history of cirrhosis or HCC"
+            }
+        }
+
+
+class ChatRequest(BaseModel):
+    """Request model for chat interactions"""
+    query: str = Field(..., description="Doctor's question about HBV guidelines", min_length=1)
+    session_id: str = Field(default="default", description="Session identifier for conversation continuity")
+    patient_context: Optional[HBVPatientInput] = Field(None, description="Optional patient context from assessment")
+    assessment_result: Optional[HBVAssessmentResponse] = Field(None, description="Optional assessment result for context")
+
+
+class ChatResponse(BaseModel):
+    """Response model for chat interactions"""
+    response: str = Field(..., description="AI response to the doctor's question")
+    session_id: str = Field(..., description="Session identifier")

api/routers/auth.py

@@ -0,0 +1,201 @@
+"""
+Authentication router for simple login system
+"""
+import os
+import secrets
+from datetime import datetime, timedelta
+from typing import Dict, Optional
+from fastapi import APIRouter, HTTPException, Response, Cookie, Form, Request
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
+import logging
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/auth", tags=["Authentication"])
+
+# Session management
+SESSION_SECRET_KEY = os.getenv("SESSION_SECRET_KEY", secrets.token_hex(32))
+SESSION_MAX_AGE = 86400  # 24 hours in seconds
+serializer = URLSafeTimedSerializer(SESSION_SECRET_KEY)
+
+# In-memory session store (for simple use case)
+# For production, consider using Redis or database
+active_sessions: Dict[str, dict] = {}
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+class LoginResponse(BaseModel):
+    success: bool
+    message: str
+
+
+def create_session(username: str) -> str:
+    """Create a new session token"""
+    session_id = secrets.token_urlsafe(32)
+    session_data = {
+        "username": username,
+        "created_at": datetime.utcnow().isoformat(),
+        "expires_at": (datetime.utcnow() + timedelta(seconds=SESSION_MAX_AGE)).isoformat()
+    }
+    
+    # Store session
+    active_sessions[session_id] = session_data
+    
+    # Create signed token
+    token = serializer.dumps(session_id)
+    return token
+
+
+def verify_session(token: Optional[str]) -> Optional[dict]:
+    """Verify session token and return session data"""
+    if not token:
+        return None
+    
+    try:
+        # Verify signature and age
+        session_id = serializer.loads(token, max_age=SESSION_MAX_AGE)
+        
+        # Check if session exists
+        session_data = active_sessions.get(session_id)
+        if not session_data:
+            return None
+        
+        # Check expiration
+        expires_at = datetime.fromisoformat(session_data["expires_at"])
+        if datetime.utcnow() > expires_at:
+            # Clean up expired session
+            active_sessions.pop(session_id, None)
+            return None
+        
+        return session_data
+    except (BadSignature, SignatureExpired):
+        return None
+    except Exception as e:
+        logger.error(f"Session verification error: {e}")
+        return None
+
+
+def verify_credentials(username: str, password: str) -> bool:
+    """Verify username and password against environment variables"""
+    expected_username = "volaris"
+    expected_password = "volaris123"
+    
+    return username == expected_username and password == expected_password
+
+
+@router.post("/login", response_model=LoginResponse)
+async def login(
+    response: Response,
+    request: Request,
+    username: str = Form(...),
+    password: str = Form(...)
+):
+    """
+    Login endpoint - validates credentials and creates session
+    """
+    # Log login attempt
+    logger.info(f"Login attempt for username: {username}, Origin: {request.headers.get('origin')}")
+    
+    # Verify credentials
+    if not verify_credentials(username, password):
+        logger.warning(f"Failed login attempt for username: {username}")
+        raise HTTPException(status_code=401, detail="Invalid username or password")
+    
+    # Create session
+    token = create_session(username)
+    logger.info(f"Session created for user: {username}")
+    
+    # Set secure cookie
+    # Detect if we're running on HTTPS (Hugging Face Spaces use HTTPS)
+    is_https = request.url.scheme == "https" or request.headers.get("x-forwarded-proto") == "https"
+    
+    # For HTTPS (production/HF Spaces), use SameSite=None with Secure=True for cross-origin
+    # For HTTP (local dev), use SameSite=Lax with Secure=False
+    if is_https:
+        samesite = "none"
+        secure = True
+    else:
+        samesite = "lax"
+        secure = False
+    
+    logger.info(f"Setting cookie with samesite={samesite}, secure={secure}, is_https={is_https}")
+
+    response.set_cookie(
+        key="session_token",
+        value=token,
+        httponly=True,
+        max_age=SESSION_MAX_AGE,
+        samesite=samesite,
+        secure=secure,
+        path="/"
+    )
+    
+    logger.info(f"Successful login for user: {username}")
+    
+    return LoginResponse(
+        success=True,
+        message="Login successful"
+    )
+
+
+@router.post("/logout")
+async def logout(
+    response: Response,
+    session_token: Optional[str] = Cookie(None)
+):
+    """
+    Logout endpoint - invalidates session
+    """
+    if session_token:
+        try:
+            session_id = serializer.loads(session_token, max_age=SESSION_MAX_AGE)
+            active_sessions.pop(session_id, None)
+        except Exception:
+            pass
+    
+    # Clear cookie
+    response.delete_cookie(key="session_token")
+    
+    return {"success": True, "message": "Logged out successfully"}
+
+
+@router.get("/verify")
+async def verify(session_token: Optional[str] = Cookie(None)):
+    """
+    Verify if current session is valid
+    """
+    session_data = verify_session(session_token)
+    
+    if not session_data:
+        raise HTTPException(status_code=401, detail="Not authenticated")
+    
+    return {
+        "authenticated": True,
+        "username": session_data.get("username")
+    }
+
+
+@router.get("/status")
+async def status(request: Request, session_token: Optional[str] = Cookie(None)):
+    """
+    Check authentication status without raising exception
+    """
+    logger.info(f"Status check - Cookie present: {session_token is not None}, Origin: {request.headers.get('origin')}")
+    session_data = verify_session(session_token)
+    
+    if session_data:
+        logger.info(f"Status check - Authenticated as: {session_data.get('username')}")
+    else:
+        logger.info("Status check - Not authenticated")
+    
+    return {
+        "authenticated": session_data is not None,
+        "username": session_data.get("username") if session_data else None
+    }

api/routers/hbv_assessment.py

@@ -0,0 +1,116 @@
+"""
+HBV Patient Assessment Router
+API endpoint for HBV treatment eligibility assessment
+"""
+from fastapi import APIRouter, HTTPException
+from api.models import HBVPatientInput, HBVAssessmentResponse, TextAssessmentInput
+import logging
+from core.hbv_assessment import assess_hbv_eligibility
+from core.text_parser import parse_patient_text, validate_extracted_data
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="",
+    tags=["HBV Assessment"]
+)
+
+
+@router.post("/assess", response_model=HBVAssessmentResponse)
+async def assess_patient(patient: HBVPatientInput) -> HBVAssessmentResponse:
+    """
+    Assess HBV patient eligibility for treatment according to SASLT 2021 guidelines
+    
+    This endpoint:
+    1. Validates patient data
+    2. Creates intelligent search query based on patient parameters
+    3. Retrieves relevant SASLT 2021 guidelines from vector store
+    4. Uses LLM to analyze patient against retrieved guidelines
+    5. Returns structured assessment with eligibility and comprehensive recommendations
+    
+    Returns:
+        HBVAssessmentResponse containing:
+        - eligible: Whether patient is eligible for treatment
+        - recommendations: Comprehensive narrative including eligibility determination, 
+          specific criteria met, treatment options (ETV, TDF, TAF), and special considerations,
+          with inline citations in format [SASLT 2021, Page X]
+    """
+    try:
+        logger.info(f"Assessing HBV patient: Age {patient.age}, Sex {patient.sex}, HBV DNA {patient.hbv_dna_level}")
+        
+        # Convert Pydantic model to dict for core function
+        patient_data = patient.dict()
+        
+        # Call core assessment function
+        result = assess_hbv_eligibility(patient_data)
+        
+        # Convert dict result back to Pydantic response model
+        response = HBVAssessmentResponse(**result)
+        
+        logger.info(f"Assessment complete: Eligible={response.eligible}")
+        return response
+        
+    except Exception as e:
+        logger.error(f"Error assessing patient: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error assessing patient: {str(e)}")
+
+
+@router.post("/assess/text", response_model=HBVAssessmentResponse)
+async def assess_patient_from_text(text_input: TextAssessmentInput) -> HBVAssessmentResponse:
+    """
+    Assess HBV patient eligibility from free-form text input
+    
+    This endpoint:
+    1. Parses free-form text to extract structured patient data using LLM
+    2. Validates the extracted data
+    3. Performs the same assessment as /assess endpoint
+    4. Returns structured assessment with eligibility and recommendations
+    
+    Example text input:
+    "45-year-old male patient
+    HBsAg: Positive for 12 months
+    HBV DNA: 5000 IU/mL
+    HBeAg: Positive
+    ALT: 80 U/L
+    Fibrosis stage: F2-F3
+    Necroinflammatory activity: A2
+    No extrahepatic manifestations
+    No immunosuppression
+    No coinfections (HIV, HCV, HDV)
+    No family history of cirrhosis or HCC"
+    
+    Returns:
+        HBVAssessmentResponse containing:
+        - eligible: Whether patient is eligible for treatment
+        - recommendations: Comprehensive narrative with inline citations
+    """
+    try:
+        logger.info(f"Received text input for assessment (length: {len(text_input.text_input)} characters)")
+        logger.info(f"Text input preview: {text_input.text_input[:200]}...")
+        
+        # Parse text to extract structured patient data
+        logger.info("Parsing text input to extract patient data...")
+        patient_data = parse_patient_text(text_input.text_input)
+        logger.info(f"Extracted patient data: {patient_data}")
+        
+        # Validate extracted data
+        logger.info("Validating extracted patient data...")
+        validated_data = validate_extracted_data(patient_data)
+        logger.info("Patient data validated successfully")
+        
+        # Call core assessment function with extracted data
+        logger.info("Performing HBV eligibility assessment...")
+        result = assess_hbv_eligibility(validated_data)
+        
+        # Convert dict result to Pydantic response model
+        response = HBVAssessmentResponse(**result)
+        
+        logger.info(f"Text-based assessment complete: Eligible={response.eligible}")
+        return response
+        
+    except ValueError as e:
+        logger.error(f"Validation error in text assessment: {str(e)}")
+        raise HTTPException(status_code=400, detail=f"Invalid patient data: {str(e)}")
+    except Exception as e:
+        logger.error(f"Error in text-based assessment: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error processing text input: {str(e)}")

api/routers/health.py

@@ -0,0 +1,228 @@
+"""
+Health Check and System Status Router
+"""
+from datetime import datetime
+from fastapi import APIRouter
+import sys
+import os
+
+# Add src to path for imports
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+
+from api.models import HealthStatus, InitializationStatus
+
+router = APIRouter(prefix="/health", tags=["health"])
+
+
+@router.get("/", response_model=HealthStatus)
+async def health_check():
+    """
+    Check the health status of the API and its components
+    """
+    components = {}
+    
+    # Check agent availability
+    try:
+        from agent import safe_run_agent
+        components["agent"] = "healthy"
+    except Exception:
+        components["agent"] = "unhealthy"
+    
+    # Check vector store
+    try:
+        from vector_store import VectorStore
+        components["vector_store"] = "healthy"
+    except Exception:
+        components["vector_store"] = "unhealthy"
+    
+    # Check data loaders
+    try:
+        from data_loaders import load_pdf_documents
+        components["data_loaders"] = "healthy"
+    except Exception:
+        components["data_loaders"] = "unhealthy"
+    
+    # Check tools
+    try:
+        from tools import medical_guidelines_knowledge_tool
+        components["tools"] = "healthy"
+    except Exception:
+        components["tools"] = "unhealthy"
+    
+    # Check initialization status
+    initialization_status = None
+    try:
+        from background_init import (
+            is_initialization_complete, 
+            get_initialization_status, 
+            is_initialization_successful,
+            get_initialization_error
+        )
+        
+        initialization_status = InitializationStatus(
+            is_complete=is_initialization_complete(),
+            status_message=get_initialization_status(),
+            is_successful=is_initialization_successful(),
+            error=str(get_initialization_error()) if get_initialization_error() else None
+        )
+    except Exception as e:
+        initialization_status = InitializationStatus(
+            is_complete=False,
+            status_message=f"Unable to check initialization status: {str(e)}",
+            is_successful=False,
+            error=str(e)
+        )
+    
+    # Overall status
+    overall_status = "healthy" if all(
+        status == "healthy" for status in components.values()
+    ) else "degraded"
+    
+    return HealthStatus(
+        status=overall_status,
+        version="1.0.0",
+        timestamp=datetime.now().isoformat(),
+        components=components,
+        initialization=initialization_status
+    )
+
+
+@router.get("/ping")
+async def ping():
+    """
+    Simple ping endpoint for basic connectivity check
+    """
+    return {"message": "pong", "timestamp": datetime.now().isoformat()}
+
+
+@router.get("/initialization", response_model=InitializationStatus)
+async def get_initialization_status():
+    """
+    Get the current initialization status of background components
+    """
+    try:
+        from background_init import (
+            is_initialization_complete, 
+            get_initialization_status, 
+            is_initialization_successful,
+            get_initialization_error
+        )
+        
+        return InitializationStatus(
+            is_complete=is_initialization_complete(),
+            status_message=get_initialization_status(),
+            is_successful=is_initialization_successful(),
+            error=str(get_initialization_error()) if get_initialization_error() else None
+        )
+    except Exception as e:
+        return InitializationStatus(
+            is_complete=False,
+            status_message=f"Unable to check initialization status: {str(e)}",
+            is_successful=False,
+            error=str(e)
+        )
+
+
+@router.get("/version")
+async def get_version():
+    """
+    Get API version information
+    """
+    return {
+        "version": "1.0.0",
+        "name": "Medical RAG AI Advisor API",
+        "description": "Professional API for medical information retrieval and advisory services",
+        "build_date": "2024-01-01"
+    }
+
+
+@router.get("/sessions")
+async def get_active_sessions():
+    """
+    Get list of all active conversation sessions
+    """
+    try:
+        from core.agent import get_active_sessions
+        sessions = get_active_sessions()
+        return {
+            "active_sessions": sessions,
+            "count": len(sessions),
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        return {
+            "error": str(e),
+            "active_sessions": [],
+            "count": 0
+        }
+
+
+@router.delete("/sessions/{session_id}")
+async def clear_session(session_id: str):
+    """
+    Clear conversation memory for a specific session
+    
+    Args:
+        session_id: The session identifier to clear
+    """
+    try:
+        from core.agent import clear_session_memory
+        success = clear_session_memory(session_id)
+        if success:
+            return {
+                "message": f"Session '{session_id}' cleared successfully",
+                "session_id": session_id,
+                "timestamp": datetime.now().isoformat()
+            }
+        else:
+            return {
+                "message": f"Session '{session_id}' not found",
+                "session_id": session_id,
+                "timestamp": datetime.now().isoformat()
+            }
+    except Exception as e:
+        return {
+            "error": str(e),
+            "session_id": session_id
+        }
+
+
+@router.delete("/sessions")
+async def clear_all_sessions():
+    """
+    Clear all conversation sessions
+    """
+    try:
+        from core.agent import clear_memory
+        clear_memory()
+        return {
+            "message": "All sessions cleared successfully",
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        return {
+            "error": str(e)
+        }
+
+
+@router.get("/sessions/{session_id}/summary")
+async def get_session_summary(session_id: str):
+    """
+    Get conversation history summary for a specific session
+    
+    Args:
+        session_id: The session identifier
+    """
+    try:
+        from core.agent import get_memory_summary
+        summary = get_memory_summary(session_id)
+        return {
+            "session_id": session_id,
+            "summary": summary,
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        return {
+            "error": str(e),
+            "session_id": session_id
+        }

api/routers/medical.py

@@ -0,0 +1,294 @@
+"""
+Medical Query Router for RAG AI Advisor
+"""
+import asyncio
+import logging
+from fastapi import APIRouter, HTTPException, status
+from fastapi.responses import StreamingResponse
+import sys
+import os
+import json
+
+# Add src to path for imports
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+
+from core.agent import safe_run_agent, safe_run_agent_streaming, clear_session_memory, get_active_sessions
+from api.models import ChatRequest, ChatResponse, HBVPatientInput, HBVAssessmentResponse
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+router = APIRouter(tags=["medical"])
+
+
+def _build_contextual_query(
+    query: str,
+    patient_context: Optional[HBVPatientInput] = None,
+    assessment_result: Optional[HBVAssessmentResponse] = None
+) -> str:
+    """
+    Build an enhanced query that includes patient context and assessment results.
+    
+    This helps the agent provide more relevant answers by understanding the specific
+    patient case being discussed.
+    
+    Args:
+        query: The doctor's original question
+        patient_context: Optional patient data from assessment
+        assessment_result: Optional assessment result with eligibility and recommendations
+    
+    Returns:
+        Enhanced query string with context
+    """
+    if not patient_context and not assessment_result:
+        # No context, return original query
+        return query
+    
+    context_parts = [query]
+    
+    # Add patient context if available
+    if patient_context:
+        context_parts.append("\n\n[PATIENT CONTEXT FOR THIS QUESTION]")
+        context_parts.append(f"- Age: {patient_context.age}, Sex: {patient_context.sex}")
+        context_parts.append(f"- HBsAg: {patient_context.hbsag_status}, HBeAg: {patient_context.hbeag_status}")
+        context_parts.append(f"- HBV DNA: {patient_context.hbv_dna_level:,.0f} IU/mL")
+        context_parts.append(f"- ALT: {patient_context.alt_level} U/L")
+        context_parts.append(f"- Fibrosis: {patient_context.fibrosis_stage}")
+        
+        if patient_context.pregnancy_status == "Pregnant":
+            context_parts.append(f"- Pregnancy: {patient_context.pregnancy_status}")
+        
+        if patient_context.immunosuppression_status and patient_context.immunosuppression_status != "None":
+            context_parts.append(f"- Immunosuppression: {patient_context.immunosuppression_status}")
+        
+        if patient_context.coinfections:
+            context_parts.append(f"- Coinfections: {', '.join(patient_context.coinfections)}")
+    
+    # Add assessment result if available
+    if assessment_result:
+        context_parts.append("\n[PRIOR ASSESSMENT RESULT]")
+        context_parts.append(f"- Eligible for treatment: {assessment_result.eligible}")
+        # Include brief summary of recommendations (first 200 chars)
+        rec_summary = assessment_result.recommendations[:200] + "..." if len(assessment_result.recommendations) > 200 else assessment_result.recommendations
+        context_parts.append(f"- Assessment summary: {rec_summary}")
+    
+    return "\n".join(context_parts)
+
+
+@router.post("/ask", response_model=ChatResponse)
+async def ask(request: ChatRequest):
+    """
+    Interactive chat endpoint for doctors to ask questions about HBV guidelines.
+    
+    This endpoint:
+    1. Accepts doctor's questions about HBV treatment guidelines
+    2. Maintains conversation context via session_id
+    3. Optionally includes patient context from prior assessment
+    4. Uses the same SASLT 2021 guidelines vector store as /assess
+    5. Returns evidence-based answers with guideline citations
+    
+    Args:
+        request: ChatRequest containing query, session_id, and optional patient/assessment context
+    
+    Returns:
+        ChatResponse with AI answer and session_id
+    """
+    try:
+        # Validate input
+        if not request.query or not request.query.strip():
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Query cannot be empty"
+            )
+        
+        if len(request.query) > 2000:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Query is too long. Maximum length is 2000 characters."
+            )
+        
+        logger.info(f"Processing chat request - Session: {request.session_id}, Query length: {len(request.query)}")
+        
+        # Build enhanced query with context if provided
+        enhanced_query = _build_contextual_query(
+            query=request.query,
+            patient_context=request.patient_context,
+            assessment_result=request.assessment_result
+        )
+        
+        # Process through agent with session context
+        response = await safe_run_agent(
+            user_input=enhanced_query,
+            session_id=request.session_id
+        )
+        
+        if not response or not response.strip():
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Received empty response from AI agent"
+            )
+        
+        logger.info(f"Chat request completed - Session: {request.session_id}")
+        
+        return ChatResponse(
+            response=response,
+            session_id=request.session_id
+        )
+        
+    except HTTPException:
+        # Re-raise HTTP exceptions as-is
+        raise
+    except Exception as e:
+        logger.error(f"Error processing chat request: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Error processing medical query: {str(e)}"
+        )
+
+
+@router.post("/ask/stream")
+async def ask_stream(request: ChatRequest):
+    """
+    Interactive streaming chat endpoint for doctors to ask questions about HBV guidelines.
+    
+    This endpoint:
+    1. Streams AI responses in real-time for better UX
+    2. Accepts doctor's questions about HBV treatment guidelines
+    3. Maintains conversation context via session_id
+    4. Optionally includes patient context from prior assessment
+    5. Uses the same SASLT 2021 guidelines vector store as /assess
+    6. Returns evidence-based answers with guideline citations
+    
+    Args:
+        request: ChatRequest containing query, session_id, and optional patient/assessment context
+    
+    Returns:
+        StreamingResponse with markdown-formatted AI answer
+    """
+    # Validate input before starting stream
+    try:
+        if not request.query or not request.query.strip():
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Query cannot be empty"
+            )
+        
+        if len(request.query) > 2000:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Query is too long. Maximum length is 2000 characters."
+            )
+        
+        logger.info(f"Processing streaming chat request - Session: {request.session_id}, Query length: {len(request.query)}")
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Validation error in streaming chat: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid request: {str(e)}"
+        )
+    
+    async def event_stream():
+        try:
+            # Build enhanced query with context if provided
+            enhanced_query = _build_contextual_query(
+                query=request.query,
+                patient_context=request.patient_context,
+                assessment_result=request.assessment_result
+            )
+            
+            chunk_buffer = ""
+            chunk_count = 0
+            
+            async for chunk in safe_run_agent_streaming(
+                user_input=enhanced_query,
+                session_id=request.session_id
+            ):
+                chunk_buffer += chunk
+                chunk_count += 1
+                
+                # Send chunks in reasonable sizes for smoother streaming
+                if len(chunk_buffer) >= 10:
+                    yield chunk_buffer
+                    chunk_buffer = ""
+                    await asyncio.sleep(0.01)
+            
+            # Send any remaining content
+            if chunk_buffer:
+                yield chunk_buffer
+            
+            logger.info(f"Streaming chat completed - Session: {request.session_id}, Chunks: {chunk_count}")
+                
+        except Exception as e:
+            error_msg = f"\n\n**Error**: An error occurred while processing your request. Please try again or contact support if the issue persists."
+            logger.error(f"Error in streaming chat: {str(e)}", exc_info=True)
+            yield error_msg
+    
+    return StreamingResponse(event_stream(), media_type="text/markdown")
+
+
+@router.delete("/session/{session_id}")
+async def clear_session(session_id: str):
+    """
+    Clear conversation history for a specific session.
+    
+    This is useful when:
+    - Starting a new patient case
+    - Switching between different patient discussions
+    - Resetting the conversation context
+    
+    Args:
+        session_id: The session identifier to clear
+    
+    Returns:
+        Success message with session status
+    """
+    try:
+        logger.info(f"Clearing session: {session_id}")
+        
+        success = clear_session_memory(session_id)
+        
+        if success:
+            return {
+                "status": "success",
+                "message": f"Session '{session_id}' cleared successfully",
+                "session_id": session_id
+            }
+        else:
+            return {
+                "status": "not_found",
+                "message": f"Session '{session_id}' not found or already cleared",
+                "session_id": session_id
+            }
+    
+    except Exception as e:
+        logger.error(f"Error clearing session {session_id}: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Error clearing session: {str(e)}"
+        )
+
+
+@router.get("/sessions")
+async def list_sessions():
+    """
+    List all active chat sessions.
+    
+    Returns:
+        List of active session IDs
+    """
+    try:
+        sessions = get_active_sessions()
+        return {
+            "status": "success",
+            "active_sessions": sessions,
+            "count": len(sessions)
+        }
+    
+    except Exception as e:
+        logger.error(f"Error listing sessions: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Error listing sessions: {str(e)}"
+        )

api/tempCodeRunnerFile.py

@@ -0,0 +1 @@
+/assess (POST)

app.py

@@ -0,0 +1,23 @@
+"""
+Startup script for HBV AI Assistant API
+"""
+import sys
+import os
+import uvicorn
+
+# Add core to Python path
+sys.path.append(os.path.join(os.path.dirname(__file__), 'core'))
+
+if __name__ == "__main__":
+    # Get port from environment variable (Hugging Face uses PORT env var)
+    port = int(os.environ.get("PORT", 7860))
+    
+    uvicorn.run(
+        "api.app:app",
+        host="0.0.0.0",  # Bind to all interfaces for deployment
+        port=port,
+        reload=False,  # Disable reload in production for faster startup
+        log_level="info",
+        access_log=True,
+        workers=1  # Single worker for Hugging Face Spaces
+    )

core/agent.py

@@ -0,0 +1,993 @@
+import logging
+import traceback
+from typing import Any, AsyncGenerator
+import asyncio
+import requests
+import os
+import httpx
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.schema import OutputParserException
+from langchain.callbacks.base import BaseCallbackHandler
+from openai import RateLimitError, APIError
+
+from .config import get_llm, logger
+from .tools import (
+    medical_guidelines_knowledge_tool,
+    get_current_datetime_tool,
+)
+
+# LangSmith tracing utilities
+from .tracing import traceable, trace, conversation_tracker, log_to_langsmith
+from .validation import validate_medical_answer
+
+
+# ============================================================================
+# STREAMING CALLBACK HANDLER
+# ============================================================================
+
+class StreamingCallbackHandler(BaseCallbackHandler):
+    """Custom callback handler for streaming responses."""
+    
+    def __init__(self):
+        self.tokens = []
+        self.current_response = ""
+    
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> Any:
+        """Called when a new token is generated."""
+        self.tokens.append(token)
+        self.current_response += token
+    
+    def get_response(self) -> str:
+        """Get the current response."""
+        return self.current_response
+    
+    def reset(self):
+        """Reset the handler for a new response."""
+        self.tokens = []
+        self.current_response = ""
+
+
+# ============================================================================
+# CUSTOM EXCEPTION CLASSES
+# ============================================================================
+
+class AgentError(Exception):
+    """Base exception for agent-related errors."""
+    pass
+
+
+class ToolExecutionError(AgentError):
+    """Exception raised when a tool fails to execute."""
+    pass
+
+
+class APIConnectionError(AgentError):
+    """Exception raised when API connections fail."""
+    pass
+
+
+class ValidationError(AgentError):
+    """Exception raised when input validation fails."""
+    pass
+
+
+# ============================================================================
+# AGENT CONFIGURATION
+# ============================================================================
+
+# Available tools for the agent
+AVAILABLE_TOOLS = [
+    medical_guidelines_knowledge_tool,
+    get_current_datetime_tool,
+]
+
+
+# System message template for the agent
+SYSTEM_MESSAGE = """
+You are a specialized HBV Clinical Assistant based on SASLT 2021 guidelines serving hepatologists and infectious disease specialists.
+
+**DUAL ROLE:**
+1. **Patient Eligibility Assessment**: Evaluate HBV patients for antiviral treatment eligibility
+2. **Clinical Consultation**: Answer questions about HBV management, guidelines, and patient cases
+
+**ASSESSMENT PARAMETERS:**
+Evaluate treatment eligibility by analyzing: Serological Status (HBsAg, HBeAg, Anti-HBe), Viral Load (HBV DNA IU/mL), Liver Function (ALT), Fibrosis Stage (F0-F4), Necroinflammatory Activity (A0-A3), Patient Category (immune tolerant/active, inactive carrier, HBeAg-negative CHB), and Special Populations (pregnancy, immunosuppression, coinfections, cirrhosis).
+
+**TREATMENT CRITERIA & OPTIONS:**
+- Eligibility: HBV DNA thresholds, ALT elevation (>ULN, >2×ULN), fibrosis stage (≥F2, F3-F4), special populations
+- First-line: Entecavir (ETV), Tenofovir Disoproxil Fumarate (TDF), Tenofovir Alafenamide (TAF)
+- Alternative agents and PEG-IFN when indicated
+
+**RESPONSE STYLE:**
+- Start directly with clinical answers - NO procedural statements ("I will retrieve...", "Let me search...", "Please wait...")
+- Use structured, concise clinical format: brief introductory sentence → organized data (tables, bullets, or structured lists) → clinical notes where relevant
+- Target 200-400 words (standard queries) or 400-600 words (complex questions)
+- Prioritize key information first, use hierarchical formatting (headers, bullets), include specific clinical parameters when relevant
+- Use precise medical terminology appropriate for experts
+- Answer only what's asked - no tangential information
+
+**STRUCTURED CLINICAL FORMAT (FOLLOW THESE EXAMPLES):**
+
+Example 1 - Tabular data with clinical notes:
+"Chronic HBV is classified into five phases using HBsAg/HBeAg status, HBV DNA, ALT, and liver inflammation.
+
+Phase 1 – Chronic HBV infection ("immune tolerant")
+HBsAg/HBeAg: High / Positive
+HBV DNA: High (>10⁷ IU/mL)
+ALT: Normal
+Liver inflammation: None/minimal
+Notes: More common/prolonged with perinatal/early-life infection; patients are highly contagious
+[SASLT HBV Management Guidelines, 2021, p. 3]
+
+Phase 2 – Chronic hepatitis B ("immune reactive HBeAg positive")
+HBsAg/HBeAg: High–intermediate / Positive
+HBV DNA: Lower (10⁴–10⁷ IU/mL)
+ALT: Increased
+Liver inflammation: Moderate/severe
+Notes: May follow years of immune tolerance; more frequent when infection occurs in adulthood
+[SASLT HBV Management Guidelines, 2021, p. 3]"
+
+Example 2 - Bullet lists with citations:
+"Screen high-risk populations despite universal childhood vaccination [SASLT 2021, p. 4].
+
+High-risk groups include:
+• Expatriate individuals (pre-employment) [SASLT 2021, p. 4]
+• Healthcare workers [SASLT 2021, p. 4]
+• Household contacts of HBV carriers [SASLT 2021, p. 4]
+• Sexual contacts of HBV carriers or those with high-risk sexual behavior [SASLT 2021, p. 4]"
+
+Example 3 - Categorized information:
+"Diagnosis of chronic HBV
+• HBsAg: detection is the most commonly used test to diagnose chronic HBV infection [SASLT 2021, p. 4].
+• HBV disease assessment incorporates HBsAg, HBeAg/anti-HBe, and HBV DNA [SASLT 2021, p. 3].
+
+Identify immunity or prior exposure
+• anti-HBs: indicates the patient is protected (immune) against HBV [SASLT 2021, p. 4].
+• anti-HBc: indicates previous exposure to HBV [SASLT 2021, p. 4]."
+
+**CLINICAL TONE & NUANCE:**
+- Use clinically precise language: "characterized by," "indicates," "reflects," "can be difficult to distinguish"
+- Acknowledge clinical uncertainty when present in guidelines: "many fall into a 'grey area'," "requires individualized follow-up," "cannot be captured by a single measurement"
+- Include practical guidance: "Practical approach recommended by the guideline," "Bottom line"
+- Add clinical context in Notes or footnotes when relevant to interpretation
+- Use specific numeric ranges and thresholds exactly as stated in guidelines (e.g., ">10⁷ IU/mL," "10⁴–10⁷ IU/mL," "≥2,000 IU/mL")
+
+**MANDATORY TOOL USAGE:**
+ALWAYS use "medical_guidelines_knowledge_tool" FIRST for every medical question - even basic HBV concepts. Do NOT answer from general knowledge. Only formulate responses based on retrieved SASLT 2021 guideline information. All information must come from SASLT 2021 (the only provider available). If no information found, explicitly state this
+
+**TOOL USAGE REQUIREMENTS:**
+1. **MEDICAL QUESTIONS** (definitions, treatments, guidelines, etc.): 
+   - MANDATORY: Use "medical_guidelines_knowledge_tool" FIRST
+   - Then answer based ONLY on retrieved information
+   
+2. **TIME/DATE QUERIES**: For current date/time or references like "today" or "now":
+   - MANDATORY: Use "get_current_datetime_tool"
+
+**SEARCH QUERY OPTIMIZATION:**
+Transform user questions into comprehensive queries with medical terminology, synonyms, clinical context, AND practical keywords. System uses hybrid search (vector + BM25 keyword matching).
+
+**Key Principles:**
+1. **Core Concept**: Start with main medical concept and guideline reference
+2. **Add Synonyms**: Include medical term variations
+3. **Add Action Verbs**: Include practical keywords from the question (e.g., "screened", "testing", "monitoring", "detection")
+4. **Expand Concepts**: Add related clinical terms
+5. **Keyword Boosters**: Append domain-specific terms at end for better coverage
+
+**Synonym Mapping:**
+- "HBV" + "hepatitis B virus" + "CHB" + "chronic hepatitis B"
+- "treatment" + "therapy" + "antiviral" + "management"
+- "ALT" + "alanine aminotransferase" + "liver enzymes"
+- "fibrosis" + "cirrhosis" + "F2 F3 F4" + "liver fibrosis"
+- "HBeAg" + "hepatitis B e antigen" + "HBeAg-positive" + "HBeAg-negative"
+- "viral load" + "HBV DNA" + "DNA level" + "viremia"
+- "screening" + "screened" + "testing" + "detection" + "diagnosis" + "program"
+
+**Concept Expansion:**
+- Treatment → "eligibility criteria indications thresholds when to start"
+- Drugs → "first-line second-line alternatives dosing monitoring ETV TDF TAF entecavir tenofovir"
+- Assessment → "HBsAg HBeAg anti-HBe HBV DNA ALT fibrosis immune phase"
+- Special populations → "pregnancy pregnant women cirrhosis immunosuppression HIV HCV HDV"
+- Screening → "target populations high-risk groups screened testing HBsAg detection program"
+
+**Query Construction Formula:**
+[Main Concept] + [Guideline Reference] + [Synonyms] + [Action Verbs from Question] + [Related Clinical Terms] + [Keyword Boosters]
+
+**Examples:**
+- "Who should be targeted for HBV screening in Saudi Arabia?" → "HBV screening target populations Saudi Arabia SASLT 2021 guidelines screened testing high-risk groups pregnancy HBsAg detection program hepatitis B virus"
+
+- "When to start treatment?" → "HBV treatment initiation criteria indications when to start SASLT 2021 HBV DNA threshold ALT elevation fibrosis stage antiviral therapy eligibility hepatitis B virus management"
+
+- "First-line drugs?" → "first-line antiviral agents HBV treatment SASLT 2021 entecavir ETV tenofovir TDF TAF preferred drugs nucleos(t)ide analogues therapy recommendations hepatitis B virus"
+
+- "HBeAg-negative management?" → "HBeAg-negative chronic hepatitis B CHB management SASLT 2021 treatment criteria HBV DNA threshold ALT elevation anti-HBe immune active phase monitoring hepatitis B e antigen"
+
+**CRITICAL**: Always include practical action verbs from the user's question (e.g., "screened", "tested", "monitored", "detected") as these improve retrieval of relevant guideline sections discussing those specific activities.
+**CITATION FORMAT (MANDATORY):**
+1. **Inline Citations**: Use format [SASLT 2021, p. X] or [SASLT HBV Management Guidelines, 2021, p. X] after each clinical statement. Cite each page individually - NEVER use ranges.
+   - Examples: 
+     * "HBsAg detection is the most commonly used test [SASLT 2021, p. 4]."
+     * "Phase 1 – Chronic HBV infection ("immune tolerant") [SASLT HBV Management Guidelines, 2021, p. 3]"
+     * "Treatment criteria include viral load thresholds [SASLT 2021, p. 7], ALT elevation [SASLT 2021, p. 8], and fibrosis assessment [SASLT 2021, p. 9]."
+
+2. **Citation Placement**: Place citation immediately after the relevant statement or at the end of each bullet point/phase description. For structured data (phases, categories), cite after each complete section.
+
+3. **References Section** (Optional): For complex answers, you may end with "**References**" listing all cited pages:
+   ```
+   **References**
+   SASLT 2021 Guidelines - Pages: p. 7, p. 8, p. 9, p. 12, p. 15, p. 18
+   (Treatment Eligibility Criteria, First-Line Agents, and Monitoring Protocols)
+   ```
+
+4. **Citation Details**: For tables/flowcharts, specify number, title, and relevant rows/columns/values. For text, specify section hierarchy. Include context pages if they contributed to your answer
+
+**NO GENERAL KNOWLEDGE - GUIDELINE ONLY:**
+NEVER answer from general knowledge or speculate. If information not found in SASLT 2021 after using tool, respond: "I searched the SASLT 2021 guidelines but could not find specific information about [topic]. You may want to rephrase with more clinical details, consult the guidelines directly, or contact hepatology specialists."
+
+**OUT-OF-SCOPE HANDLING:**
+For non-HBV questions (other diseases, non-medical topics), respond professionally: "I'm unable to assist with that request, but I'd be happy to help with HBV-related inquiries."
+
+**PATIENT CONTEXT:**
+When question includes [PATIENT CONTEXT] or [PRIOR ASSESSMENT RESULT], provide personalized case-specific guidance tailored to patient's parameters (HBV DNA, ALT, fibrosis). Reference prior assessments for consistency.
+
+**ELIGIBILITY ASSESSMENT WORKFLOW:**
+1. Retrieve SASLT 2021 criteria via tool
+2. Categorize patient phase (immune tolerant/active, inactive carrier, HBeAg-negative CHB, cirrhosis)
+3. Compare parameters: HBV DNA vs. threshold, ALT vs. ULN, fibrosis stage, necroinflammatory activity
+4. Check special considerations (pregnancy, immunosuppression, coinfections, HCC family history)
+5. Determine eligibility (Eligible/Not Eligible/Borderline)
+6. Recommend first-line agents (ETV, TDF, TAF) if eligible
+7. Outline monitoring plan
+
+**ELIGIBILITY RESPONSE STRUCTURE:**
+For patient eligibility: Patient Profile (HBsAg, HBeAg, HBV DNA, ALT, Fibrosis) → SASLT 2021 Criteria (with page citations) → Eligibility Status (Eligible/Not Eligible/Borderline) + Rationale → Treatment Recommendations (ETV, TDF, TAF if eligible) → Monitoring → References
+
+**FORMATTING:**
+Use **bold** for critical points/drugs, headers (###) for organization, bullets/numbered lists for sequences, tables for comparisons, blockquotes (>) for direct quotes. Include specific numeric values and thresholds.
+
+**SAFETY:**
+For emergencies (acute liver failure, hepatic encephalopathy, severe bleeding, loss of consciousness), respond: "This is an emergency! Call emergency services immediately and seek urgent medical help." Educational information only - not a substitute for clinical judgment. Always respond in English.
+"""
+
+# Create the prompt template
+prompt_template = ChatPromptTemplate.from_messages([
+    ("system", SYSTEM_MESSAGE),
+    MessagesPlaceholder("chat_history"),
+    ("human", "{input}"),
+    MessagesPlaceholder("agent_scratchpad"),
+])
+
+# Initialize the agent with lazy loading
+def get_agent():
+    """Get agent with lazy loading for faster startup"""
+    return create_openai_tools_agent(
+        llm=get_llm(),
+        tools=AVAILABLE_TOOLS,
+        prompt=prompt_template,
+    )
+
+# Create agent executor with lazy loading
+def get_agent_executor():
+    """Get agent executor with lazy loading for faster startup"""
+    return AgentExecutor(
+        agent=get_agent(),
+        tools=AVAILABLE_TOOLS,
+        verbose=True,
+        handle_parsing_errors=True,
+        max_iterations=5,
+        max_execution_time=90,  # tighten a bit to help responsiveness
+    )
+
+# ============================================================================
+# SESSION-BASED MEMORY MANAGEMENT
+# ============================================================================
+
+class SessionMemoryManager:
+    """Manages conversation memory for multiple sessions."""
+    
+    def __init__(self):
+        self._sessions = {}
+        self._default_window_size = 20  # Increased from 10 to maintain better context
+    
+    def get_memory(self, session_id: str = "default") -> ConversationBufferWindowMemory:
+        """Get or create memory for a specific session."""
+        if session_id not in self._sessions:
+            import warnings
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", category=DeprecationWarning)
+                self._sessions[session_id] = ConversationBufferWindowMemory(
+                    memory_key="chat_history",
+                    return_messages=True,
+                    max_window_size=self._default_window_size
+                )
+        return self._sessions[session_id]
+    
+    def clear_session(self, session_id: str) -> bool:
+        """Clear memory for a specific session."""
+        if session_id in self._sessions:
+            self._sessions[session_id].clear()
+            del self._sessions[session_id]
+            return True
+        return False
+    
+    def clear_all_sessions(self):
+        """Clear all session memories."""
+        for memory in self._sessions.values():
+            memory.clear()
+        self._sessions.clear()
+    
+    def get_active_sessions(self) -> list:
+        """Get list of active session IDs."""
+        return list(self._sessions.keys())
+
+# Global session memory manager
+_memory_manager = SessionMemoryManager()
+
+
+# ============================================================================
+# VALIDATION HELPER FUNCTIONS
+# ============================================================================
+
+def _should_validate_response(user_input: str, response: str) -> bool:
+    """
+    Determine if a response should be automatically validated.
+    
+    Args:
+        user_input: The user's input
+        response: The agent's response
+        
+    Returns:
+        bool: True if the response should be validated
+    """
+    # Skip validation for certain types of responses
+    skip_indicators = [
+        "side effect report",
+        "adverse drug reaction report",
+        "error:",
+        "sorry,",
+        "i don't know",
+        "i do not know",
+        "could not find specific information",
+        "not found in the retrieved guidelines",
+        "validation report",
+        "evaluation scores"
+    ]
+    
+    # Skip validation for side effect reporting queries in user input
+    side_effect_input_indicators = [
+        "side effect", "adverse reaction", "adverse event", "drug reaction",
+        "medication reaction", "patient experienced", "developed after taking",
+        "caused by medication", "drug-related", "medication-related"
+    ]
+    
+    user_input_lower = user_input.lower()
+    response_lower = response.lower()
+    
+    # Don't validate if user input is about side effect reporting
+    if any(indicator in user_input_lower for indicator in side_effect_input_indicators):
+        return False
+    
+    # Don't validate if response contains skip indicators
+    if any(indicator in response_lower for indicator in skip_indicators):
+        return False
+    
+    # Don't validate very short responses
+    if len(response.strip()) < 50:
+        return False
+    
+    # Validate if response seems to contain medical information
+    medical_indicators = [
+        "treatment", "therapy", "diagnosis", "medication", "drug", "patient",
+        "clinical", "guideline", "recommendation", "according to", "source:",
+        "provider:", "page:", "saslt", "hbv", "hepatitis"
+    ]
+    
+    return any(indicator in response_lower for indicator in medical_indicators)
+
+
+def _perform_automatic_validation(user_input: str, response: str) -> None:
+    """
+    Perform automatic validation in the background without displaying results to user.
+    Validation results are logged and saved to GitHub repository for backend analysis.
+    
+    Args:
+        user_input: The user's input
+        response: The agent's response
+        
+    Returns:
+        None: Validation runs silently in background
+    """
+    try:
+        # Import here to avoid circular imports
+        from .tools import _last_question, _last_documents
+        
+        # Check if we have the necessary context for validation
+        if not _last_question or not _last_documents:
+            logger.info("Skipping validation: insufficient context")
+            return
+        
+        # Perform validation using the original user input instead of tool query
+        evaluation = validate_medical_answer(user_input, _last_documents, response)
+        
+        # Log validation results to backend only (not shown to user)
+        report = evaluation.get("validation_report", {})
+        logger.info(f"Background validation completed - Interaction ID: {evaluation.get('interaction_id', 'N/A')}")
+        logger.info(f"Validation scores - Overall: {report.get('Overall_Rating', 'N/A')}/100, "
+                   f"Accuracy: {report.get('Accuracy_Rating', 'N/A')}/100, "
+                   f"Coherence: {report.get('Coherence_Rating', 'N/A')}/100, "
+                   f"Relevance: {report.get('Relevance_Rating', 'N/A')}/100")
+        
+        # Validation is automatically saved to GitHub by validate_medical_answer function
+        # No need to return anything - results are stored in backend only
+        
+    except Exception as e:
+        logger.error(f"Background validation failed: {e}")
+
+
+# ============================================================================
+# STREAMING AGENT FUNCTIONS
+# ============================================================================
+
+# @traceable(name="run_agent_streaming")
+async def run_agent_streaming(user_input: str, session_id: str = "default", max_retries: int = 3) -> AsyncGenerator[str, None]:
+    """
+    Run the agent with streaming support and comprehensive error handling.
+    
+    This function processes user input through the agent executor with streaming
+    capabilities, robust error handling, and automatic retries for recoverable errors.
+    
+    Args:
+        user_input (str): The user's input message to process
+        session_id (str, optional): Session identifier for conversation memory. Defaults to "default".
+        max_retries (int, optional): Maximum number of retries for recoverable errors. 
+                                   Defaults to 3.
+    
+    Yields:
+        str: Chunks of the agent's response as they are generated
+        
+    Raises:
+        None: All exceptions are caught and handled internally
+    """
+    # Input validation
+    if not user_input or not user_input.strip():
+        logger.warning("Empty input received")
+        yield "Sorry, I didn't receive any questions. Please enter your question or request."
+        return
+    
+    retry_count = 0
+    last_error = None
+    current_run_id = None
+    # Session metadata (increment conversation count)
+    session_metadata = conversation_tracker.get_session_metadata(increment=True)
+    
+    while retry_count <= max_retries:
+        try:
+            # Tracing for streaming disabled to avoid duplicate traces.
+            # We keep tracing only for the AgentExecutor in run_agent().
+            current_run_id = None
+            # Load conversation history from session-specific memory
+            memory = _memory_manager.get_memory(session_id)
+            chat_history = memory.load_memory_variables({})["chat_history"]
+            
+            logger.info(f"Processing user input (attempt {retry_count + 1}): {user_input[:50]}...")
+            
+            # Create streaming callback handler
+            streaming_handler = StreamingCallbackHandler()
+            
+            # Run the agent in a separate thread to avoid blocking
+            def run_sync():
+                return get_agent_executor().invoke(
+                    {
+                        "input": user_input.strip(),
+                        "chat_history": chat_history,
+                    },
+                    config={"callbacks": [streaming_handler]},
+                )
+            
+            # Execute the agent with streaming
+            full_response = ""
+            previous_length = 0
+            
+            # Start the agent execution in background
+            loop = asyncio.get_event_loop()
+            task = loop.run_in_executor(None, run_sync)
+            
+            # Stream the response as it's being generated
+            while not task.done():
+                current_response = streaming_handler.get_response()
+                
+                # Yield new tokens if available
+                if len(current_response) > previous_length:
+                    new_content = current_response[previous_length:]
+                    previous_length = len(current_response)
+                    yield new_content
+                
+                # Small delay to prevent overwhelming the client (faster flushing)
+                await asyncio.sleep(0.03)
+            
+            # Get the final result
+            response = await task
+            
+            # Yield any remaining content
+            final_response = streaming_handler.get_response()
+            if len(final_response) > previous_length:
+                yield final_response[previous_length:]
+            
+            # If no streaming content was captured, yield the full response
+            if not final_response and response and "output" in response:
+                full_output = response["output"]
+                # Simulate streaming by yielding word by word
+                words = full_output.split(' ')
+                for word in words:
+                    yield word + ' '
+                    await asyncio.sleep(0.05)
+                final_response = full_output
+            
+            # Validate response structure
+            if not response or "output" not in response:
+                raise ValidationError("Invalid response format from agent")
+            
+            if not response["output"] or not response["output"].strip():
+                raise ValidationError("Empty response from agent")
+            
+            # Perform automatic validation in background (hidden from user)
+            base_response = response["output"]
+            if _should_validate_response(user_input, base_response):
+                logger.info("Performing background validation for streaming response...")
+                try:
+                    # Run validation silently - results saved to backend/GitHub only
+                    _perform_automatic_validation(user_input, base_response)
+                except Exception as e:
+                    logger.error(f"Background validation failed: {e}")
+            
+            # Save conversation context to memory
+            memory.save_context(
+                {"input": user_input},
+                {"output": response["output"]}
+            )
+            
+            # Log response metrics to LangSmith
+            try:
+                log_to_langsmith(
+                    key="response_metrics",
+                    value={
+                        "response_length": len(response.get("output", "")),
+                        "attempt": retry_count + 1,
+                        **session_metadata,
+                    },
+                    run_id=current_run_id,
+                )
+            except Exception:
+                pass
+
+            logger.info(f"Successfully processed user input: {user_input[:50]}...")
+            return
+            
+        except RateLimitError as e:
+            retry_count += 1
+            last_error = e
+            wait_time = min(2 ** retry_count, 60)  # Exponential backoff, max 60 seconds
+            
+            logger.warning(
+                f"Rate limit exceeded. Retrying in {wait_time} seconds... "
+                f"(Attempt {retry_count}/{max_retries})"
+            )
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(wait_time)
+                continue
+            else:
+                logger.error("Rate limit exceeded after maximum retries")
+                yield "Sorry, the system is currently busy. Please try again in a little while."
+                return
+                
+        except (APIError, httpx.RemoteProtocolError, httpx.ReadError, httpx.ConnectError) as e:
+            retry_count += 1
+            last_error = e
+            error_type = type(e).__name__
+            logger.error(f"OpenAI API/Connection error ({error_type}): {str(e)}")
+            
+            if retry_count <= max_retries:
+                wait_time = min(2 ** retry_count, 10)  # Exponential backoff, max 10 seconds
+                logger.info(f"Retrying after {wait_time} seconds... (Attempt {retry_count}/{max_retries})")
+                await asyncio.sleep(wait_time)
+                continue
+            else:
+                yield "Sorry, there was an error connecting to the service. Please try again later."
+                return
+                
+        except requests.exceptions.ConnectionError as e:
+            retry_count += 1
+            last_error = e
+            logger.error(f"Network connection error: {str(e)}")
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(3)
+                continue
+            else:
+                yield "Sorry, I can't connect to the service right now. Please check your internet connection and try again."
+                return
+                
+        except requests.exceptions.Timeout as e:
+            retry_count += 1
+            last_error = e
+            logger.error(f"Request timeout: {str(e)}")
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(2)
+                continue
+            else:
+                yield "Sorry, the request took longer than expected. Please try again."
+                return
+                
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request error: {str(e)}")
+            yield "Sorry, an error occurred with the request. Please try again."
+            return
+            
+        except OutputParserException as e:
+            logger.error(f"Output parsing error: {str(e)}")
+            yield "Sorry, an error occurred while processing the response. Please rephrase your question and try again."
+            return
+            
+        except ValidationError as e:
+            logger.error(f"Validation error: {str(e)}")
+            yield "Sorry, an error occurred while validating the data. Please try again."
+            return
+            
+        except ToolExecutionError as e:
+            logger.error(f"Tool execution error: {str(e)}")
+            yield "Sorry, an error occurred while executing one of the operations. Please try again or contact technical support."
+            return
+            
+        except Exception as e:
+            logger.error(f"Unexpected error in run_agent_streaming: {str(e)}")
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            # Log error to LangSmith
+            try:
+                log_to_langsmith(
+                    key="error_log",
+                    value={
+                        "error": str(e),
+                        "error_type": type(e).__name__,
+                        **session_metadata,
+                    },
+                    run_id=current_run_id,
+                )
+            except Exception:
+                pass
+            
+            # For unexpected errors, don't retry
+            yield "Sorry, an unexpected error occurred. Please try again or contact technical support if the problem persists."
+            return
+    
+    # This should never be reached, but just in case
+    logger.error(f"Maximum retries exceeded. Last error: {str(last_error)}")
+    yield "Sorry, I was unable to process your request after several attempts. Please try again later."
+
+
+async def safe_run_agent_streaming(user_input: str, session_id: str = "default") -> AsyncGenerator[str, None]:
+    """
+    Streaming wrapper function with additional safety checks and input validation.
+    
+    This function provides an additional layer of safety by validating input parameters,
+    checking input length constraints, and handling any critical errors that might
+    occur during streaming agent execution.
+    
+    Args:
+        user_input (str): The user's input message to process
+        session_id (str, optional): Session identifier for conversation memory. Defaults to "default".
+        
+    Yields:
+        str: Chunks of the agent's response as they are generated
+        
+    Raises:
+        None: All exceptions are caught and handled internally
+    """
+    try:
+        # Input type validation
+        if not isinstance(user_input, str):
+            logger.warning(f"Invalid input type received: {type(user_input)}")
+            yield "Sorry, the input must be valid text."
+            return
+        
+        # Input length validation
+        stripped_input = user_input.strip()
+        
+        if len(stripped_input) > 1000:
+            logger.warning(f"Input too long: {len(stripped_input)} characters")
+            yield "Sorry, the message is too long. Please shorten your question."
+            return
+        
+        if len(stripped_input) == 0:
+            logger.warning("Empty input after stripping")
+            yield "Sorry, I didn't receive any questions. Please enter your question or request."
+            return
+        
+        # Stream the response through the main agent function
+        async for chunk in run_agent_streaming(user_input, session_id):
+            yield chunk
+        
+    except Exception as e:
+        logger.critical(f"Critical error in safe_run_agent_streaming: {str(e)}")
+        logger.critical(f"Traceback: {traceback.format_exc()}")
+        yield "Sorry, a critical system error occurred. Please contact technical support immediately."
+
+
+@traceable(name="run_agent")
+async def run_agent(user_input: str, session_id: str = "default", max_retries: int = 3) -> str:
+    """
+    Run the agent with comprehensive error handling and retry logic.
+    
+    This function processes user input through the agent executor with robust
+    error handling, automatic retries for recoverable errors, and comprehensive
+    logging for debugging and monitoring.
+    
+    Args:
+        user_input (str): The user's input message to process
+        session_id (str, optional): Session identifier for conversation memory. Defaults to "default".
+        max_retries (int, optional): Maximum number of retries for recoverable errors. 
+                                   Defaults to 3.
+    
+    Returns:
+        str: The agent's response or an appropriate error message in English
+        
+    Raises:
+        None: All exceptions are caught and handled internally
+    """
+    # Input validation
+    if not user_input or not user_input.strip():
+        logger.warning("Empty input received")
+        return "Sorry, I didn't receive any questions. Please enter your question or request."
+    
+    retry_count = 0
+    last_error = None
+    current_run_id = None
+    session_metadata = conversation_tracker.get_session_metadata(increment=True)
+    
+    while retry_count <= max_retries:
+        try:
+            # Load conversation history from session-specific memory
+            memory = _memory_manager.get_memory(session_id)
+            chat_history = memory.load_memory_variables({})["chat_history"]
+
+            logger.info(f"Processing user input (attempt {retry_count + 1}): {user_input[:50]}...")
+
+            # Invoke the agent with input and history (synchronous call)
+            response = get_agent_executor().invoke({
+                "input": user_input.strip(),
+                "chat_history": chat_history
+            })
+            current_run_id = None  # This will be handled by LangChain's tracer
+            
+            # Validate response structure
+            if not response or "output" not in response or not isinstance(response["output"], str):
+                raise ValidationError("Invalid response format from agent")
+            
+            if not response["output"] or not response["output"].strip():
+                raise ValidationError("Empty response from agent")
+            
+            # Save conversation context to memory
+            memory.save_context(
+                {"input": user_input},
+                {"output": response["output"]}
+            )
+            
+            # Log response metrics
+            try:
+                log_to_langsmith(
+                    key="response_metrics",
+                    value={
+                        "response_length": len(response.get("output", "")),
+                        "attempt": retry_count + 1,
+                        **session_metadata,
+                    },
+                    run_id=current_run_id,
+                )
+            except Exception:
+                pass
+
+            logger.info(f"Successfully processed user input: {user_input[:50]}...")
+            
+            # Perform automatic validation in background (hidden from user)
+            final_response = response["output"]
+            if _should_validate_response(user_input, final_response):
+                logger.info("Performing background validation...")
+                try:
+                    # Run validation silently - results saved to backend/GitHub only
+                    _perform_automatic_validation(user_input, final_response)
+                except Exception as e:
+                    logger.error(f"Background validation failed: {e}")
+            
+            return final_response
+            
+        except RateLimitError as e:
+            retry_count += 1
+            last_error = e
+            wait_time = min(2 ** retry_count, 60)  # Exponential backoff, max 60 seconds
+            
+            logger.warning(
+                f"Rate limit exceeded. Retrying in {wait_time} seconds... "
+                f"(Attempt {retry_count}/{max_retries})"
+            )
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(wait_time)
+                continue
+            else:
+                logger.error("Rate limit exceeded after maximum retries")
+                return "Sorry, the system is currently busy. Please try again in a little while."
+                
+        except APIError as e:
+            retry_count += 1
+            last_error = e
+            logger.error(f"OpenAI API error: {str(e)}")
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(2)
+                continue
+            else:
+                return "Sorry, there was an error connecting to the service. Please try again later."
+                
+        except requests.exceptions.ConnectionError as e:
+            retry_count += 1
+            last_error = e
+            logger.error(f"Network connection error: {str(e)}")
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(3)
+                continue
+            else:
+                return "Sorry, I can't connect to the service right now. Please check your internet connection and try again."
+                
+        except requests.exceptions.Timeout as e:
+            retry_count += 1
+            last_error = e
+            logger.error(f"Request timeout: {str(e)}")
+            
+            if retry_count <= max_retries:
+                await asyncio.sleep(2)
+                continue
+            else:
+                return "Sorry, the request took longer than expected. Please try again."
+                
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request error: {str(e)}")
+            return "Sorry, an error occurred with the request. Please try again."
+            
+        except OutputParserException as e:
+            logger.error(f"Output parsing error: {str(e)}")
+            return "Sorry, an error occurred while processing the response. Please rephrase your question and try again."
+            
+        except ValidationError as e:
+            logger.error(f"Validation error: {str(e)}")
+            return "Sorry, an error occurred while validating the data. Please try again."
+            
+        except ToolExecutionError as e:
+            logger.error(f"Tool execution error: {str(e)}")
+            return "Sorry, an error occurred while executing one of the operations. Please try again or contact technical support."
+            
+        except Exception as e:
+            logger.error(f"Unexpected error in run_agent: {str(e)}")
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            # Log error
+            try:
+                log_to_langsmith(
+                    key="error_log",
+                    value={
+                        "error": str(e),
+                        "error_type": type(e).__name__,
+                        **session_metadata,
+                    },
+                    run_id=current_run_id,
+                )
+            except Exception:
+                pass
+            
+            # For unexpected errors, don't retry
+            return "Sorry, an unexpected error occurred. Please try again or contact technical support if the problem persists."
+    
+    # This should never be reached, but just in case
+    logger.error(f"Maximum retries exceeded. Last error: {str(last_error)}")
+    return "Sorry, I was unable to process your request after several attempts. Please try again later."
+
+
+async def safe_run_agent(user_input: str, session_id: str = "default") -> str:
+    """
+    Wrapper function for run_agent with additional safety checks and input validation.
+    
+    This function provides an additional layer of safety by validating input parameters,
+    checking input length constraints, and handling any critical errors that might
+    occur during agent execution.
+    
+    Args:
+        user_input (str): The user's input message to process
+        session_id (str, optional): Session identifier for conversation memory. Defaults to "default".
+        
+    Returns:
+        str: The agent's response or an appropriate error message in English
+        
+    Raises:
+        None: All exceptions are caught and handled internally
+    """
+    try:
+        # Input type validation
+        if not isinstance(user_input, str):
+            logger.warning(f"Invalid input type received: {type(user_input)}")
+            return "Sorry, the input must be valid text."
+        
+        # Input length validation
+        stripped_input = user_input.strip()
+        
+        # if len(stripped_input) > 1000:
+        #     logger.warning(f"Input too long: {len(stripped_input)} characters")
+        #     return "Sorry, the message is too long. Please shorten your question."
+        
+        if len(stripped_input) == 0:
+            logger.warning("Empty input after stripping")
+            return "Sorry, I didn't receive any questions. Please enter your question or request."
+        
+        # Process the input through the main agent function
+        return await run_agent(user_input, session_id)
+        
+    except Exception as e:
+        logger.critical(f"Critical error in safe_run_agent: {str(e)}")
+        logger.critical(f"Traceback: {traceback.format_exc()}")
+        return "Sorry, a critical system error occurred. Please contact technical support immediately."
+
+
+def clear_memory() -> None:
+    """
+    Clear the conversation memory.
+    
+    This function clears all stored conversation history from memory,
+    effectively starting a fresh conversation session.
+    """
+    try:
+        _memory_manager.clear_all_sessions()
+        logger.info("Conversation memory cleared successfully")
+    except Exception as e:
+        logger.error(f"Error clearing memory: {str(e)}")
+
+
+def get_memory_summary(session_id: str = "default") -> str:
+    """
+    Get a summary of the conversation history for a specific session.
+    
+    Args:
+        session_id (str, optional): Session identifier. Defaults to "default".
+    
+    Returns:
+        str: A summary of the conversation history stored in memory
+    """
+    try:
+        memory = _memory_manager.get_memory(session_id)
+        memory_vars = memory.load_memory_variables({})
+        return str(memory_vars.get("chat_history", "No conversation history available"))
+    except Exception as e:
+        logger.error(f"Error getting memory summary: {str(e)}")
+        return "Error retrieving conversation history"
+
+
+def clear_session_memory(session_id: str) -> bool:
+    """
+    Clear conversation memory for a specific session.
+    
+    Args:
+        session_id (str): Session identifier to clear
+    
+    Returns:
+        bool: True if session was cleared, False if session didn't exist
+    """
+    return _memory_manager.clear_session(session_id)
+
+
+def get_active_sessions() -> list:
+    """
+    Get list of all active session IDs.
+    
+    Returns:
+        list: List of active session identifiers
+    """
+    return _memory_manager.get_active_sessions()

core/background_init.py

@@ -0,0 +1,153 @@
+"""
+Background initialization system for preloading heavy components during app startup.
+This module handles the eager loading of embedding models, retrievers, and chunks
+to improve first-question response time.
+"""
+
+import threading
+import time
+from typing import Optional, Callable
+from .config import logger
+
+
+class BackgroundInitializer:
+    """Manages background initialization of heavy components"""
+    
+    def __init__(self):
+        self._initialization_thread: Optional[threading.Thread] = None
+        self._initialization_complete = threading.Event()
+        self._initialization_error: Optional[Exception] = None
+        self._progress_callback: Optional[Callable[[str, int], None]] = None
+        self._status = "Not started"
+        
+    def set_progress_callback(self, callback: Callable[[str, int], None]):
+        """Set a callback function to receive progress updates"""
+        self._progress_callback = callback
+        
+    def _update_progress(self, message: str, percentage: int):
+        """Update progress and call callback if set"""
+        self._status = message
+        logger.info(f"🔄 Background Init: {message} ({percentage}%)")
+        if self._progress_callback:
+            try:
+                self._progress_callback(message, percentage)
+            except Exception as e:
+                logger.error(f"Progress callback error: {e}")
+    
+    def _initialize_components(self):
+        """Initialize all heavy components in background thread"""
+        try:
+            self._update_progress("Starting background initialization...", 0)
+            
+            # Step 1: Load embedding model (this is the heaviest component)
+            self._update_progress("Loading embedding model...", 10)
+            from .config import get_embedding_model
+            embedding_model = get_embedding_model()
+            self._update_progress("Embedding model loaded successfully", 40)
+            
+            # Step 2: Initialize retrievers (this will load chunks and create vector store)
+            self._update_progress("Initializing retrievers and loading chunks...", 50)
+            from .retrievers import _ensure_initialized
+            _ensure_initialized()
+            self._update_progress("Retrievers initialized successfully", 90)
+            
+            # Step 3: Learn medical terminology from corpus
+            self._update_progress("Learning medical terminology from corpus...", 92)
+            try:
+                from .medical_terminology import learn_from_corpus
+                from . import utils
+                
+                # Load chunks to learn from
+                chunks = utils.load_chunks()
+                if chunks:
+                    # Convert to format expected by learner
+                    documents = [{'content': chunk.page_content} for chunk in chunks[:1000]]  # Limit for performance
+                    learn_from_corpus(documents)
+                    logger.info(f"Learned medical terminology from {len(documents)} documents")
+            except Exception as e:
+                logger.warning(f"Could not learn terminology from corpus: {e}")
+            
+            # Step 4: Warm up LLM (optional, lightweight)
+            self._update_progress("Warming up LLM...", 97)
+            from .config import get_llm
+            llm = get_llm()
+            self._update_progress("All components initialized successfully", 100)
+            
+            logger.info("✅ Background initialization completed successfully")
+            
+        except Exception as e:
+            self._initialization_error = e
+            logger.error(f"❌ Background initialization failed: {e}")
+            self._update_progress(f"Initialization failed: {str(e)}", -1)
+        finally:
+            self._initialization_complete.set()
+    
+    def start_background_initialization(self):
+        """Start background initialization in a separate thread"""
+        if self._initialization_thread is not None:
+            logger.warning("Background initialization already started")
+            return
+            
+        logger.info("🚀 Starting background initialization...")
+        self._initialization_thread = threading.Thread(
+            target=self._initialize_components,
+            name="BackgroundInitializer",
+            daemon=True
+        )
+        self._initialization_thread.start()
+    
+    def is_complete(self) -> bool:
+        """Check if initialization is complete"""
+        return self._initialization_complete.is_set()
+    
+    def wait_for_completion(self, timeout: Optional[float] = None) -> bool:
+        """Wait for initialization to complete"""
+        return self._initialization_complete.wait(timeout)
+    
+    def get_status(self) -> str:
+        """Get current initialization status"""
+        return self._status
+    
+    def get_error(self) -> Optional[Exception]:
+        """Get initialization error if any"""
+        return self._initialization_error
+    
+    def is_successful(self) -> bool:
+        """Check if initialization completed successfully"""
+        return self.is_complete() and self._initialization_error is None
+
+
+# Global initializer instance
+_background_initializer = BackgroundInitializer()
+
+
+def start_background_initialization(progress_callback: Optional[Callable[[str, int], None]] = None):
+    """Start background initialization with optional progress callback"""
+    if progress_callback:
+        _background_initializer.set_progress_callback(progress_callback)
+    _background_initializer.start_background_initialization()
+
+
+def wait_for_initialization(timeout: Optional[float] = None) -> bool:
+    """Wait for background initialization to complete"""
+    return _background_initializer.wait_for_completion(timeout)
+
+
+def is_initialization_complete() -> bool:
+    """Check if background initialization is complete"""
+    return _background_initializer.is_complete()
+
+
+def get_initialization_status() -> str:
+    """Get current initialization status"""
+    return _background_initializer.get_status()
+
+
+def get_initialization_error() -> Optional[Exception]:
+    """Get initialization error if any"""
+    return _background_initializer.get_error()
+
+
+def is_initialization_successful() -> bool:
+    """Check if initialization completed successfully"""
+    return _background_initializer.is_successful()

core/config.py

@@ -0,0 +1,152 @@
+import os
+from pathlib import Path
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_openai import ChatOpenAI
+from dotenv import load_dotenv
+import logging
+from logging.handlers import RotatingFileHandler
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+# Initialize environment
+load_dotenv()
+
+
+# --- Settings (simple, in-file) ---
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore')
+
+    OPENAI_API_KEY: str
+    OPENAI_BASE_URL: str | None = None
+
+    LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
+    DATA_DIR: str = os.getenv("DATA_DIR", "")
+    LOG_DIR: str = os.getenv("LOG_DIR", "")
+
+
+settings = Settings()
+
+
+# --- File Path Configuration (Cross-platform compatible) ---
+PROJECT_ROOT = Path(__file__).parent.parent.absolute()
+DATA_DIR = Path(settings.DATA_DIR or (PROJECT_ROOT / "data"))
+NEW_DATA = DATA_DIR / "new_data"
+PROCESSED_DATA = DATA_DIR / "processed_data"
+CHUNKS_PATH = DATA_DIR / "chunks.pkl"
+VECTOR_STORE_DIR = DATA_DIR / "vector_store"
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+NEW_DATA.mkdir(parents=True, exist_ok=True)
+PROCESSED_DATA.mkdir(parents=True, exist_ok=True)
+VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True)
+
+# Setup logging
+LOG_DIR = Path(settings.LOG_DIR or (Path(__file__).parent.parent / "logs"))
+LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+LOG_FILE = LOG_DIR / "app.log"
+
+# Configure application logger (avoid duplicate handlers)
+LOG_LEVEL = settings.LOG_LEVEL.upper()
+logger = logging.getLogger("AgenticMedicalRAG")  # centralized logger
+logger.setLevel(LOG_LEVEL)
+logger.propagate = False
+if not logger.handlers:
+    formatter = logging.Formatter(
+        fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    file_handler = RotatingFileHandler(
+        LOG_FILE,
+        maxBytes=1000000,
+        backupCount=3,
+        encoding="utf-8"
+    )
+    file_handler.setFormatter(formatter)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+    logger.addHandler(stream_handler)
+
+
+
+# --- LLM Configuration with lazy loading ---
+_llm = None
+
+def get_llm():
+    """Get LLM with lazy loading for faster startup"""
+    global _llm
+    if _llm is None:
+        logger.info("Initializing LLM (first time)...")
+        openai_key = settings.OPENAI_API_KEY
+        
+        if not openai_key:
+            logger.error("OPENAI_API_KEY not found in environment variables")
+            raise ValueError("OpenAI API key is required. Please set OPENAI_API_KEY environment variable.")
+        
+        try:
+            _llm = ChatOpenAI(
+                model="gpt-4o",
+                api_key=openai_key,
+                base_url=settings.OPENAI_BASE_URL,
+                temperature=0.0,
+                max_tokens=2048,
+                request_timeout=30,  # Increased timeout for stability
+                max_retries=2,
+                streaming=True,
+            )
+            logger.info("LLM initialized successfully")
+        except Exception as e:
+            logger.error(f"Failed to initialize LLM: {e}")
+            raise
+    return _llm
+
+def create_llm():
+    """Create LLM with proper error handling and fallbacks"""
+    return get_llm()
+
+# Lazy loading - only initialize when actually needed
+LLM = None  # Will be loaded on first use
+
+# --- Embedding Model Configuration with lazy loading ---
+_embedding_model = None
+
+def get_embedding_model():
+    """Get embedding model with lazy loading for faster startup"""
+    global _embedding_model
+    if _embedding_model is None:
+        logger.info("Loading embedding model (first time)...")
+        try:
+            _embedding_model = HuggingFaceEmbeddings(
+                model_name="abhinand/MedEmbed-base-v0.1",
+                model_kwargs={'device': 'cpu'},
+                encode_kwargs={'normalize_embeddings': True}
+            )
+            logger.info("Embedding model loaded successfully")
+        except Exception as e:
+            logger.error(f"Failed to load embedding model: {e}")
+            raise ValueError("Failed to load embedding model")
+    return _embedding_model
+
+# For backward compatibility
+def create_embedding_model():
+    """Create embedding model with proper error handling"""
+    return get_embedding_model()
+
+# Lazy loading - only load when actually needed
+EMBEDDING_MODEL = None  # Will be loaded on first use
+
+# Configuration validation
+def validate_config():
+    """Validate all required configurations"""
+    required_env_vars = ["OPENAI_API_KEY"]
+    missing_vars = [var for var in required_env_vars if not getattr(settings, var, None)]
+    
+    if missing_vars:
+        raise ValueError(f"Missing required environment variables: {missing_vars}")
+    
+    logger.info("Configuration validation completed")
+
+# Run validation on import
+try:
+    validate_config()
+except Exception as e:
+    logger.error(f"Configuration validation failed: {e}")
+    raise e

core/context_enrichment.py

@@ -0,0 +1,342 @@
+"""
+Context Enrichment Module for Medical RAG
+
+This module enriches retrieved documents with surrounding context (adjacent pages)
+to provide comprehensive information for expert medical professionals.
+"""
+
+from typing import List, Dict, Set, Optional
+from langchain.schema import Document
+from pathlib import Path
+from .config import logger
+
+
+class ContextEnricher:
+    """
+    Enriches retrieved documents with surrounding pages for richer context.
+    """
+    
+    def __init__(self, cache_size: int = 100):
+        """
+        Initialize context enricher with document cache.
+        
+        Args:
+            cache_size: Maximum number of source documents to cache
+        """
+        self._document_cache: Dict[str, List[Document]] = {}
+        self._cache_size = cache_size
+        self._all_chunks_cache: Optional[List[Document]] = None  # Cache all chunks to avoid reloading
+    
+    def enrich_documents(
+        self,
+        retrieved_docs: List[Document],
+        pages_before: int = 1,
+        pages_after: int = 1,
+        max_enriched_docs: int = 5
+    ) -> List[Document]:
+        """
+        Enrich retrieved documents by adding separate context pages.
+        
+        Args:
+            retrieved_docs: List of retrieved documents
+            pages_before: Number of pages to include before each document
+            pages_after: Number of pages to include after each document
+            max_enriched_docs: Maximum number of documents to enrich (top results)
+        
+        Returns:
+            List with original documents + separate context page documents
+        """
+        if not retrieved_docs:
+            return []
+        
+        result_docs = []
+        processed_sources = set()
+        enriched_count = 0
+        
+        # Only enrich top documents to avoid overwhelming context
+        docs_to_enrich = retrieved_docs[:max_enriched_docs]
+        
+        for doc in docs_to_enrich:
+            try:
+                # Get source information
+                source = doc.metadata.get('source', 'unknown')
+                page_num = doc.metadata.get('page_number', 1)
+                
+                # Skip if already processed this source-page combination
+                source_page_key = f"{source}_{page_num}"
+                if source_page_key in processed_sources:
+                    continue
+                
+                processed_sources.add(source_page_key)
+                
+                # Get surrounding pages
+                surrounding_docs = self._get_surrounding_pages(
+                    doc, 
+                    pages_before, 
+                    pages_after
+                )
+                
+                if surrounding_docs:
+                    # Add separate documents for each page
+                    page_docs = self._create_separate_page_documents(
+                        doc, 
+                        surrounding_docs,
+                        pages_before,
+                        pages_after
+                    )
+                    result_docs.extend(page_docs)
+                    enriched_count += 1
+                    
+                    # Log enrichment details
+                    page_numbers = [int(d.metadata.get('page_number', 0)) for d in page_docs]
+                    logger.debug(f"Enriched {source} page {page_num} with pages: {page_numbers}")
+                else:
+                    # No surrounding pages found, add original with empty enrichment metadata
+                    original_with_metadata = self._add_empty_enrichment_metadata(doc)
+                    result_docs.append(original_with_metadata)
+                    
+            except Exception as e:
+                logger.warning(f"Could not enrich document from {doc.metadata.get('source')}: {e}")
+                original_with_metadata = self._add_empty_enrichment_metadata(doc)
+                result_docs.append(original_with_metadata)
+        
+        # Add remaining documents without enrichment
+        for doc in retrieved_docs[max_enriched_docs:]:
+            original_with_metadata = self._add_empty_enrichment_metadata(doc)
+            result_docs.append(original_with_metadata)
+        
+        logger.info(f"Enriched {enriched_count} documents with surrounding context pages")
+        return result_docs
+    
+    def _get_surrounding_pages(
+        self,
+        doc: Document,
+        pages_before: int,
+        pages_after: int
+    ) -> List[Document]:
+        """
+        Get surrounding pages for a document.
+        
+        Args:
+            doc: Original document
+            pages_before: Number of pages before
+            pages_after: Number of pages after
+        
+        Returns:
+            List of surrounding documents (including original), deduplicated by page number
+        """
+        source = doc.metadata.get('source', 'unknown')
+        page_num = doc.metadata.get('page_number', 1)
+        provider = doc.metadata.get('provider', 'unknown')
+        disease = doc.metadata.get('disease', 'unknown')
+        
+        # Try to get full document from cache or load it
+        full_doc_pages = self._get_full_document(source, provider, disease)
+        
+        if not full_doc_pages:
+            return []
+        
+        # Find the target page and surrounding pages
+        target_page = int(page_num) if isinstance(page_num, (int, str)) else 1
+        
+        # Use a dict to deduplicate by page number (keep first occurrence)
+        pages_dict = {}
+        
+        for page_doc in full_doc_pages:
+            doc_page_num = page_doc.metadata.get('page_number', 0)
+            if isinstance(doc_page_num, str):
+                try:
+                    doc_page_num = int(doc_page_num)
+                except:
+                    continue
+            
+            # Include pages within range
+            if target_page - pages_before <= doc_page_num <= target_page + pages_after:
+                # Only add if not already present (deduplication)
+                if doc_page_num not in pages_dict:
+                    pages_dict[doc_page_num] = page_doc
+        
+        # Return sorted by page number
+        surrounding = [pages_dict[pn] for pn in sorted(pages_dict.keys())]
+        
+        return surrounding
+    
+    def _get_full_document(
+        self,
+        source: str,
+        provider: str,
+        disease: str
+    ) -> Optional[List[Document]]:
+        """
+        Get full document pages from chunks cache.
+        
+        Args:
+            source: Source filename
+            provider: Provider name
+            disease: Disease name
+        
+        Returns:
+            List of all pages in the document, or None if not found
+        """
+        cache_key = f"{provider}_{disease}_{source}"
+        
+        # Check cache
+        if cache_key in self._document_cache:
+            return self._document_cache[cache_key]
+        
+        # Load from chunks cache instead of trying to reload PDFs
+        try:
+            from . import utils
+            
+            # Load all chunks (use cached version to avoid redundant loading)
+            if self._all_chunks_cache is None:
+                self._all_chunks_cache = utils.load_chunks()
+                if self._all_chunks_cache:
+                    logger.debug(f"Loaded {len(self._all_chunks_cache)} chunks into enricher cache")
+            
+            all_chunks = self._all_chunks_cache
+            if not all_chunks:
+                logger.debug(f"No chunks available for enrichment")
+                return None
+            
+            # Filter chunks for this specific document
+            doc_pages = []
+            for chunk in all_chunks:
+                chunk_source = chunk.metadata.get('source', '')
+                chunk_provider = chunk.metadata.get('provider', '')
+                chunk_disease = chunk.metadata.get('disease', '')
+                
+                # Match by source, provider, and disease
+                if (chunk_source == source and 
+                    chunk_provider == provider and 
+                    chunk_disease == disease):
+                    doc_pages.append(chunk)
+            
+            if not doc_pages:
+                logger.debug(f"Could not find chunks for document: {source} (Provider: {provider}, Disease: {disease})")
+                return None
+            
+            # Sort by page number
+            doc_pages.sort(key=lambda d: int(d.metadata.get('page_number', 0)))
+            
+            # Cache it (with size limit)
+            if len(self._document_cache) >= self._cache_size:
+                # Remove oldest entry
+                self._document_cache.pop(next(iter(self._document_cache)))
+            
+            self._document_cache[cache_key] = doc_pages
+            logger.debug(f"Loaded {len(doc_pages)} pages for {source} from chunks cache")
+            return doc_pages
+            
+        except Exception as e:
+            logger.warning(f"Error loading document from chunks cache {source}: {e}")
+            return None
+    
+    def _create_separate_page_documents(
+        self,
+        original_doc: Document,
+        surrounding_docs: List[Document],
+        pages_before: int,
+        pages_after: int
+    ) -> List[Document]:
+        """
+        Create separate document objects for original page and context pages.
+        
+        Args:
+            original_doc: Original retrieved document
+            surrounding_docs: List of surrounding documents
+            pages_before: Number of pages before
+            pages_after: Number of pages after
+        
+        Returns:
+            List of separate documents (context pages + original page + context pages)
+        """
+        # Sort by page number
+        sorted_docs = sorted(
+            surrounding_docs,
+            key=lambda d: int(d.metadata.get('page_number', 0))
+        )
+        
+        original_page = int(original_doc.metadata.get('page_number', 1))
+        result_docs = []
+        
+        for doc in sorted_docs:
+            page_num = int(doc.metadata.get('page_number', 0))
+            
+            # Determine if this is a context page or the original page
+            is_context_page = (page_num != original_page)
+            
+            # Create document with appropriate metadata
+            page_doc = Document(
+                page_content=doc.page_content,
+                metadata={
+                    **doc.metadata,
+                    'context_enrichment': is_context_page,
+                    'enriched': False,
+                    'pages_included': [],
+                    'primary_page': None,
+                    'context_pages_before': None,
+                    'context_pages_after': None,
+                }
+            )
+            
+            result_docs.append(page_doc)
+        
+        return result_docs
+    
+    def _add_empty_enrichment_metadata(self, doc: Document) -> Document:
+        """
+        Add empty enrichment metadata fields to a document.
+        
+        Args:
+            doc: Original document
+        
+        Returns:
+            Document with enrichment metadata fields set to default values
+        """
+        return Document(
+            page_content=doc.page_content,
+            metadata={
+                **doc.metadata,
+                'enriched': False,
+                'pages_included': [],
+                'primary_page': None,
+                'context_pages_before': None,
+                'context_pages_after': None,
+            }
+        )
+
+
+# Global enricher instance
+_context_enricher = ContextEnricher(cache_size=100)
+
+
+def enrich_retrieved_documents(
+    documents: List[Document],
+    pages_before: int = 1,
+    pages_after: int = 1,
+    max_enriched: int = 5
+) -> List[Document]:
+    """
+    Convenience function to enrich retrieved documents.
+    
+    Args:
+        documents: Retrieved documents
+        pages_before: Number of pages to include before each document
+        pages_after: Number of pages to include after each document
+        max_enriched: Maximum number of documents to enrich
+    
+    Returns:
+        Enriched documents with surrounding context
+    """
+    return _context_enricher.enrich_documents(
+        documents,
+        pages_before=pages_before,
+        pages_after=pages_after,
+        max_enriched_docs=max_enriched
+    )
+
+
+def get_context_enricher() -> ContextEnricher:
+    """Get the global context enricher instance."""
+    return _context_enricher

core/data_loaders.py

@@ -0,0 +1,142 @@
+# Import required libraries
+import pandas as pd
+from pathlib import Path
+from typing import List
+from langchain.schema import Document
+from .config import logger
+from langchain_pymupdf4llm import PyMuPDF4LLMLoader
+from langchain_community.document_loaders.parsers import TesseractBlobParser
+
+
+def load_pdf_documents(pdf_path: Path) -> List[Document]:
+    """
+    Load and process PDF documents from medical guidelines using PyMuPDF4LLMLoader.
+    Uses Tesseract for image extraction and optimized table extraction for medical documents.
+    Extracts disease and provider from directory structure.
+   
+    Directory structure expected: data/new_data/PROVIDER/file.pdf
+    Example: data/new_data/SASLT/SASLT_2021.pdf
+   
+    Args:
+        pdf_path: Path to the PDF file
+       
+    Returns:
+        List of Document objects with metadata (source, disease, provider, page_number)
+    """
+    try:
+       
+        # Validate file exists
+        if not pdf_path.exists():
+            raise FileNotFoundError(f"PDF file not found at {pdf_path}")
+       
+        # Extract provider from directory structure
+        # Structure: data/new_data/PROVIDER/file.pdf
+        path_parts = pdf_path.parts
+        disease = "HBV"  # Default disease for this system
+        provider = "unknown"
+       
+        # Find provider: it's the parent directory of the PDF file
+        if len(path_parts) >= 2:
+            provider = path_parts[-2]  # Parent directory (e.g., SASLT)
+            
+        # If provider is 'new_data', it means file is directly in new_data folder
+        if provider.lower() == "new_data":
+            provider = "unknown"
+       
+        # Initialize PyMuPDF4LLMLoader
+        loader = PyMuPDF4LLMLoader(
+            str(pdf_path),
+            mode="page",
+            extract_images=True,
+            images_parser=TesseractBlobParser(),
+            table_strategy="lines"
+        )
+       
+        raw_documents = loader.load()
+       
+        documents = []
+        for idx, doc in enumerate(raw_documents):
+            if doc.page_content.strip():
+                # Extract actual page number from metadata, default to sequential numbering
+                # PyMuPDF4LLMLoader uses 0-indexed pages, so we add 1 for human-readable page numbers
+                actual_page = doc.metadata.get("page")
+                if actual_page is not None:
+                    # If page is 0-indexed, add 1 to make it 1-indexed
+                    page_num = actual_page + 1 if actual_page == idx else actual_page
+                else:
+                    # Fallback to 1-indexed sequential numbering
+                    page_num = idx + 1
+                
+                processed_doc = Document(
+                    page_content=doc.page_content,
+                    metadata={
+                        "source": pdf_path.name,
+                        "disease": disease,
+                        "provider": provider,
+                        "page_number": page_num
+                    }
+                )
+                documents.append(processed_doc)
+
+        logger.info(f"Loaded {len(documents)} pages from PDF: {pdf_path.name} (Disease: {disease}, Provider: {provider})")
+        return documents
+       
+    except Exception as e:
+        logger.error(f"Error loading PDF documents from {pdf_path}: {str(e)}")
+        raise
+    
+    
+def load_markdown_documents(md_path: Path) -> List[Document]:
+    """
+    Load and process Markdown medical guidelines.
+    Extracts disease and provider from directory structure.
+
+    Directory structure expected: data/new_data/PROVIDER/file.md
+    Example: data/new_data/SASLT/guidelines.md
+
+    Args:
+        md_path: Path to the Markdown file
+
+    Returns:
+        List of Document objects with metadata (source, disease, provider, page_number)
+    """
+    try:
+        # Validate file exists
+        if not md_path.exists():
+            raise FileNotFoundError(f"Markdown file not found at {md_path}")
+
+        # Extract provider from directory structure
+        # Structure: data/new_data/PROVIDER/file.md
+        path_parts = md_path.parts
+        disease = "HBV"  # Default disease for this system
+        provider = "unknown"
+
+        # Find provider: it's the parent directory of the markdown file
+        if len(path_parts) >= 2:
+            provider = path_parts[-2]  # Parent directory (e.g., SASLT)
+            
+        # If provider is 'new_data', it means file is directly in new_data folder
+        if provider.lower() == "new_data":
+            provider = "unknown"
+
+        # Read markdown content
+        with open(md_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+
+        # Create document with minimal metadata for RAG
+        doc = Document(
+            page_content=content,
+            metadata={
+                "source": md_path.name,
+                "disease": disease,
+                "provider": provider,
+                "page_number": 1
+            }
+        )
+
+        logger.info(f"Loaded Markdown document: {md_path.name} (Disease: {disease}, Provider: {provider})")
+        return [doc]
+
+    except Exception as e:
+        logger.error(f"Error loading Markdown document from {md_path}: {str(e)}")
+        raise

core/github_storage.py

@@ -0,0 +1,462 @@
+"""
+GitHub Storage Utility for Medical RAG Advisor
+Handles saving side effects reports and validation results to GitHub repository
+"""
+import os
+import json
+import csv
+import io
+import base64
+import time
+import traceback
+from datetime import datetime
+from typing import Dict, List, Any, Optional
+import requests
+from .config import logger
+
+
+class GitHubStorage:
+    """
+    Utility class for storing medical data files in GitHub repository
+    """
+    
+    def __init__(self, repo_url: str = "https://github.com/MoazEldsouky/HBV-AI-Assistant-data", 
+                 github_token: str = None):
+        """
+        Initialize GitHub storage with repository details
+        
+        Args:
+            repo_url: GitHub repository URL (default: HBV AI Assistant data repository)
+            github_token: GitHub personal access token
+        """
+        self.repo_url = repo_url
+        self.github_token = github_token or os.getenv("GITHUB_TOKEN", "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")
+        
+        # Log token status (masked for security)
+        if self.github_token:
+            token_preview = self.github_token[:7] + "..." + self.github_token[-4:] if len(self.github_token) > 11 else "***"
+            logger.info(f"GitHub token configured: {token_preview}")
+        else:
+            logger.warning("No GitHub token configured - uploads will fail!")
+        
+        # Extract owner and repo name from URL
+        if "github.com/" in repo_url:
+            parts = repo_url.replace("https://github.com/", "").replace(".git", "").split("/")
+            self.owner = parts[0]
+            self.repo_name = parts[1]
+        else:
+            raise ValueError("Invalid GitHub repository URL format")
+        
+        self.api_base = f"https://api.github.com/repos/{self.owner}/{self.repo_name}"
+        self.headers = {
+            "Authorization": f"token {self.github_token}",
+            "Accept": "application/vnd.github.v3+json",
+            "Content-Type": "application/json"
+        }
+        
+        logger.info(f"GitHub storage initialized for {self.owner}/{self.repo_name}")
+    
+    def _get_file_sha(self, file_path: str) -> Optional[str]:
+        """
+        Get the SHA of an existing file in the repository
+        
+        Args:
+            file_path: Path to file in repository
+            
+        Returns:
+            SHA string if file exists, None otherwise
+        """
+        try:
+            url = f"{self.api_base}/contents/{file_path}"
+            response = requests.get(url, headers=self.headers)
+            
+            if response.status_code == 200:
+                return response.json().get("sha")
+            elif response.status_code == 404:
+                return None
+            else:
+                logger.error(f"Error getting file SHA: {response.status_code} - {response.text}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"Exception getting file SHA: {e}")
+            return None
+    
+    def _upload_file(self, file_path: str, content: str, message: str, sha: Optional[str] = None) -> bool:
+        """
+        Upload or update a file in the GitHub repository
+        
+        Args:
+            file_path: Path where file should be stored in repo
+            content: File content as string
+            message: Commit message
+            sha: SHA of existing file (for updates)
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Encode content to base64
+            content_encoded = base64.b64encode(content.encode('utf-8')).decode('utf-8')
+            
+            # Prepare request data
+            data = {
+                "message": message,
+                "content": content_encoded
+            }
+            
+            # Add SHA if updating existing file
+            if sha:
+                data["sha"] = sha
+            
+            # Make API request with timeout
+            url = f"{self.api_base}/contents/{file_path}"
+            logger.info(f"Uploading to GitHub: {file_path} (size: {len(content)} bytes)")
+            response = requests.put(url, headers=self.headers, json=data, timeout=30)
+            
+            if response.status_code in [200, 201]:
+                logger.info(f"✓ Successfully uploaded {file_path} to GitHub")
+                return True
+            elif response.status_code == 401:
+                logger.error(f"❌ Authentication failed uploading {file_path}: Invalid or expired GitHub token")
+                logger.error(f"Response: {response.text}")
+                return False
+            elif response.status_code == 403:
+                logger.error(f"❌ Permission denied uploading {file_path}: Token lacks required permissions")
+                logger.error(f"Response: {response.text}")
+                return False
+            elif response.status_code == 404:
+                logger.error(f"❌ Repository not found: {self.owner}/{self.repo_name}")
+                logger.error(f"Response: {response.text}")
+                return False
+            elif response.status_code == 409:
+                logger.error(f"Conflict error uploading {file_path}: File may have been modified. Status: {response.status_code}")
+                logger.error(f"Response: {response.text[:500]}")
+                return False
+            else:
+                logger.error(f"Failed to upload {file_path}. Status: {response.status_code}")
+                logger.error(f"Response: {response.text}")
+                return False
+                
+        except requests.exceptions.Timeout as e:
+            logger.error(f"Timeout uploading file to GitHub: {e}")
+            return False
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request exception uploading file to GitHub: {e}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected exception uploading file to GitHub: {e}")
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            return False
+    
+    def _get_file_content(self, file_path: str) -> Optional[str]:
+        """
+        Get the content of a file from the GitHub repository
+        
+        Args:
+            file_path: Path to file in repository
+            
+        Returns:
+            File content as string if successful, None otherwise
+        """
+        try:
+            url = f"{self.api_base}/contents/{file_path}"
+            response = requests.get(url, headers=self.headers)
+            
+            if response.status_code == 200:
+                content_encoded = response.json().get("content", "")
+                content = base64.b64decode(content_encoded).decode('utf-8')
+                return content
+            elif response.status_code == 404:
+                return None
+            else:
+                logger.error(f"Error getting file content: {response.status_code} - {response.text}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"Exception getting file content: {e}")
+            return None
+    
+    def save_side_effects_report(self, report_data: Dict[str, Any]) -> bool:
+        """
+        Save a side effects report to GitHub repository as CSV
+        
+        Args:
+            report_data: Dictionary containing side effects report data
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            file_path = "medical_data/side_effects_reports.csv"
+            
+            # Get existing file content
+            existing_content = self._get_file_content(file_path)
+            
+            # Define CSV fieldnames
+            fieldnames = [
+                'timestamp', 'drug_name', 'side_effects', 'patient_age', 
+                'patient_gender', 'dosage', 'duration', 'severity', 
+                'outcome', 'additional_details', 'reporter_info', 'raw_input'
+            ]
+            
+            # Create CSV content
+            output = io.StringIO()
+            writer = csv.DictWriter(output, fieldnames=fieldnames)
+            
+            # If file doesn't exist, write header
+            if existing_content is None:
+                writer.writeheader()
+                csv_content = output.getvalue()
+            else:
+                # File exists, append to existing content
+                csv_content = existing_content
+            
+            # Append new row
+            output = io.StringIO()
+            writer = csv.DictWriter(output, fieldnames=fieldnames)
+            writer.writerow(report_data)
+            new_row = output.getvalue()
+            
+            # Combine existing content with new row
+            final_content = csv_content + new_row
+            
+            # Get SHA for update
+            sha = self._get_file_sha(file_path)
+            
+            # Upload file
+            commit_message = f"Add side effects report for {report_data.get('drug_name', 'unknown drug')} - {report_data.get('timestamp', 'unknown time')}"
+            
+            return self._upload_file(file_path, final_content, commit_message, sha)
+            
+        except Exception as e:
+            logger.error(f"Error saving side effects report to GitHub: {e}")
+            return False
+    
+    def save_validation_results(self, evaluation_data: Dict[str, Any]) -> bool:
+        """
+        Save validation results to GitHub repository as JSON with robust append logic.
+        Always loads existing data first, then appends new evaluation without overwriting.
+        
+        Args:
+            evaluation_data: Dictionary containing evaluation data with interaction_id already set
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        max_retries = 3
+        retry_count = 0
+        
+        while retry_count < max_retries:
+            try:
+                file_path = "medical_data/evaluation_results.json"
+                
+                # STEP 1: Get existing file content with verification
+                logger.info(f"Attempt {retry_count + 1}/{max_retries}: Loading existing evaluations from GitHub...")
+                existing_content = self._get_file_content(file_path)
+                
+                # STEP 2: Parse existing data or create new list
+                evaluations = []
+                if existing_content:
+                    try:
+                        evaluations = json.loads(existing_content)
+                        if not isinstance(evaluations, list):
+                            logger.warning("Existing content is not a list, creating new list")
+                            evaluations = []
+                        else:
+                            logger.info(f"Successfully loaded {len(evaluations)} existing evaluations")
+                    except json.JSONDecodeError as e:
+                        logger.error(f"Failed to parse existing evaluation_results.json: {e}")
+                        # Don't start fresh - this could lose data. Instead, fail and retry.
+                        if retry_count < max_retries - 1:
+                            retry_count += 1
+                            logger.warning(f"Retrying due to JSON parse error...")
+                            time.sleep(2)  # Wait before retry
+                            continue
+                        else:
+                            logger.error("Max retries reached. Cannot parse existing data.")
+                            return False
+                else:
+                    logger.info("No existing file found, creating new evaluation list")
+                
+                # STEP 3: Verify we're not about to lose data
+                new_interaction_id = evaluation_data.get('interaction_id', 'unknown')
+                logger.info(f"Adding new evaluation with ID: {new_interaction_id}")
+                
+                # Check if this ID already exists (prevent duplicates)
+                existing_ids = [e.get('interaction_id') for e in evaluations]
+                if new_interaction_id in existing_ids:
+                    logger.warning(f"Evaluation with ID {new_interaction_id} already exists. Skipping duplicate.")
+                    return True  # Not an error, just already saved
+                
+                # STEP 4: Add new evaluation to the list (APPEND, not replace)
+                evaluations.append(evaluation_data)
+                logger.info(f"Appended new evaluation. Total count: {len(evaluations)}")
+                
+                # STEP 5: Convert to JSON string
+                json_content = json.dumps(evaluations, indent=2, ensure_ascii=False)
+                
+                # STEP 6: Get SHA for update (must be fresh to avoid conflicts)
+                sha = self._get_file_sha(file_path)
+                if existing_content and not sha:
+                    logger.error("File exists but SHA not found. Possible race condition.")
+                    if retry_count < max_retries - 1:
+                        retry_count += 1
+                        logger.warning("Retrying due to SHA retrieval failure...")
+                        time.sleep(2)  # Wait before retry
+                        continue
+                    else:
+                        return False
+                
+                # STEP 7: Upload file with the complete list
+                commit_message = f"Add validation results for interaction {new_interaction_id} - {evaluation_data.get('timestamp', 'unknown time')}"
+                
+                success = self._upload_file(file_path, json_content, commit_message, sha)
+                
+                if success:
+                    logger.info(f"✓ Successfully saved evaluation {new_interaction_id}. Total evaluations now: {len(evaluations)}")
+                    return True
+                else:
+                    logger.error(f"Failed to upload file (attempt {retry_count + 1}/{max_retries})")
+                    if retry_count < max_retries - 1:
+                        retry_count += 1
+                        logger.warning("Retrying upload...")
+                        time.sleep(2)  # Wait before retry
+                        continue
+                    else:
+                        return False
+                
+            except Exception as e:
+                logger.error(f"Error saving validation results to GitHub (attempt {retry_count + 1}/{max_retries}): {e}")
+                if retry_count < max_retries - 1:
+                    retry_count += 1
+                    logger.warning("Retrying due to exception...")
+                    time.sleep(2)  # Wait before retry
+                    continue
+                else:
+                    return False
+        
+        return False
+    
+    def get_side_effects_reports(self) -> List[Dict[str, Any]]:
+        """
+        Get all side effects reports from GitHub repository
+        
+        Returns:
+            List of side effects reports as dictionaries
+        """
+        try:
+            file_path = "medical_data/side_effects_reports.csv"
+            content = self._get_file_content(file_path)
+            
+            if not content:
+                return []
+            
+            # Parse CSV content
+            csv_reader = csv.DictReader(io.StringIO(content))
+            reports = list(csv_reader)
+            
+            return reports
+            
+        except Exception as e:
+            logger.error(f"Error getting side effects reports from GitHub: {e}")
+            return []
+    
+    def get_validation_results(self, limit: int = 10) -> Dict[str, Any]:
+        """
+        Get validation results from GitHub repository
+        
+        Args:
+            limit: Maximum number of recent evaluations to return
+            
+        Returns:
+            Dictionary containing evaluation summary and recent evaluations
+        """
+        try:
+            file_path = "medical_data/evaluation_results.json"
+            content = self._get_file_content(file_path)
+            
+            if not content:
+                return {"message": "No evaluations found", "evaluations": []}
+            
+            # Parse JSON content
+            evaluations = json.loads(content)
+            if not isinstance(evaluations, list):
+                evaluations = []
+            
+            # Get recent evaluations
+            recent_evaluations = evaluations[-limit:] if evaluations else []
+            
+            # Calculate average scores
+            if recent_evaluations:
+                total_scores = {
+                    "accuracy": 0,
+                    "coherence": 0,
+                    "relevance": 0,
+                    "completeness": 0,
+                    "citations": 0,
+                    "length": 0,
+                    "overall": 0
+                }
+                
+                count = len(recent_evaluations)
+                for eval_data in recent_evaluations:
+                    report = eval_data.get("validation_report", {})
+                    total_scores["accuracy"] += int(report.get("Accuracy_Rating", 0))
+                    total_scores["coherence"] += int(report.get("Coherence_Rating", 0))
+                    total_scores["relevance"] += int(report.get("Relevance_Rating", 0))
+                    total_scores["completeness"] += int(report.get("Completeness_Rating", 0))
+                    total_scores["citations"] += int(report.get("Citations_Attribution_Rating", 0))
+                    total_scores["length"] += int(report.get("Length_Rating", 0))
+                    total_scores["overall"] += int(report.get("Overall_Rating", 0))
+                
+                averages = {key: round(value / count, 1) for key, value in total_scores.items()}
+            else:
+                averages = {}
+            
+            return {
+                "total_evaluations": len(evaluations),
+                "recent_count": len(recent_evaluations),
+                "average_scores": averages,
+                "evaluations": recent_evaluations
+            }
+            
+        except Exception as e:
+            logger.error(f"Error getting validation results from GitHub: {e}")
+            return {"error": str(e), "evaluations": []}
+    
+    def get_drug_reports(self, drug_name: str) -> List[Dict[str, Any]]:
+        """
+        Get side effects reports for a specific drug from GitHub repository
+        
+        Args:
+            drug_name: Name of the drug to filter reports
+            
+        Returns:
+            List of reports for the specified drug
+        """
+        try:
+            all_reports = self.get_side_effects_reports()
+            
+            # Filter reports for the specific drug (case-insensitive)
+            drug_reports = [
+                report for report in all_reports 
+                if report.get('drug_name', '').lower() == drug_name.lower()
+            ]
+            
+            return drug_reports
+            
+        except Exception as e:
+            logger.error(f"Error getting drug reports from GitHub: {e}")
+            return []
+
+
+# Global GitHub storage instance
+_github_storage = None
+
+def get_github_storage() -> GitHubStorage:
+    """Get the global GitHub storage instance with lazy loading."""
+    global _github_storage
+    if _github_storage is None:
+        _github_storage = GitHubStorage()
+    return _github_storage

core/hbv_assessment.py

@@ -0,0 +1,298 @@
+"""
+HBV Patient Assessment Module
+Evaluates patient eligibility for HBV treatment according to SASLT 2021 guidelines
+"""
+import logging
+import json
+from typing import Dict, Any
+from .retrievers import hybrid_search
+from .config import get_llm
+
+logger = logging.getLogger(__name__)
+
+
+def create_patient_query(patient_data: Dict[str, Any]) -> str:
+    """
+    Create a comprehensive search query based on patient parameters
+    
+    Args:
+        patient_data: Dictionary containing patient clinical parameters
+        
+    Returns:
+        Optimized search query string for guideline retrieval
+    """
+    query_parts = []
+    
+    # Add HBeAg status to query
+    if patient_data.get("hbeag_status") == "Positive":
+        query_parts.append("HBeAg-positive chronic hepatitis B treatment eligibility")
+    else:
+        query_parts.append("HBeAg-negative chronic hepatitis B treatment eligibility")
+    
+    # Add viral load context
+    hbv_dna = patient_data.get("hbv_dna_level", 0)
+    if hbv_dna > 20000:
+        query_parts.append("high HBV DNA level")
+    elif hbv_dna > 2000:
+        query_parts.append("moderate HBV DNA level")
+    
+    # Add ALT context
+    sex = patient_data.get("sex", "Male")
+    alt_level = patient_data.get("alt_level", 0)
+    alt_uln = 35 if sex == "Male" else 25
+    if alt_level > 2 * alt_uln:
+        query_parts.append("significantly elevated ALT")
+    elif alt_level > alt_uln:
+        query_parts.append("elevated ALT")
+    
+    # Add fibrosis context
+    fibrosis_stage = patient_data.get("fibrosis_stage", "")
+    if fibrosis_stage == "F4":
+        query_parts.append("cirrhosis treatment criteria")
+    elif fibrosis_stage == "F2-F3":
+        query_parts.append("significant fibrosis")
+    
+    # Add special populations
+    if patient_data.get("pregnancy_status") == "Pregnant":
+        query_parts.append("pregnancy antiviral prophylaxis")
+    
+    immunosuppression = patient_data.get("immunosuppression_status")
+    if immunosuppression and immunosuppression != "None":
+        query_parts.append("immunosuppression prophylactic therapy")
+    
+    coinfections = patient_data.get("coinfections", [])
+    if coinfections:
+        query_parts.append("coinfection management")
+    
+    if patient_data.get("extrahepatic_manifestations"):
+        query_parts.append("extrahepatic manifestations")
+    
+    if patient_data.get("family_history_cirrhosis_hcc"):
+        query_parts.append("family history HCC cirrhosis")
+    
+    # Combine into search query
+    base_query = " ".join(query_parts[:3])  # Use top 3 most relevant parts
+    return f"{base_query} treatment indications criteria SASLT 2021"
+
+
+def assess_hbv_eligibility(patient_data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Assess patient eligibility for HBV treatment based on SASLT 2021 guidelines
+    using retrieval from vector store and LLM analysis
+    
+    Args:
+        patient_data: Dictionary containing patient clinical parameters
+        
+    Returns:
+        Dictionary with assessment results:
+        - eligible: bool
+        - recommendations: str (comprehensive narrative with inline citations in format [SASLT 2021, Page X])
+    """
+    try:
+        # Check if HBsAg is positive (required for treatment consideration)
+        if patient_data.get("hbsag_status") != "Positive":
+            return {
+                "eligible": False,
+                "recommendations": "Patient is HBsAg negative. HBV treatment is not indicated. HBsAg positivity is required for HBV treatment consideration according to SASLT 2021 guidelines."
+            }
+        
+        # Create search query based on patient parameters
+        search_query = create_patient_query(patient_data)
+        logger.info(f"Generated search query: {search_query}")
+        
+        # Retrieve relevant guidelines from vector store
+        docs = hybrid_search(search_query, provider="SASLT", k_vector=8, k_bm25=2)
+        
+        if not docs:
+            logger.warning("No documents retrieved from vector store")
+            return {
+                "eligible": False,
+                "recommendations": "Unable to retrieve SASLT 2021 guidelines. Please try again."
+            }
+        
+        # Log retrieved documents
+        logger.info(f"\n{'='*80}")
+        logger.info(f"RETRIEVED DOCUMENTS ({len(docs)} documents)")
+        logger.info(f"{'='*80}")
+        for i, doc in enumerate(docs, 1):
+            source = doc.metadata.get('source', 'Unknown')
+            page = doc.metadata.get('page_number', 'N/A')
+            content_preview = doc.page_content[:200] + "..." if len(doc.page_content) > 200 else doc.page_content
+            logger.info(f"\n📄 Document {i}:")
+            logger.info(f"   Source: {source}")
+            logger.info(f"   Page: {page}")
+            logger.info(f"   Content Preview: {content_preview}")
+        logger.info(f"{'='*80}\n")
+        
+        # Format retrieved documents for LLM
+        context = "\n\n".join([
+            f"[Source: {doc.metadata.get('source', 'Unknown')}, Page: {doc.metadata.get('page_number', 'N/A')}]\n{doc.page_content}"
+            for doc in docs
+        ])
+        
+        # Define ALT ULN for context
+        sex = patient_data.get("sex", "Male")
+        alt_uln = 35 if sex == "Male" else 25
+        
+        # Format patient data for prompt
+        age = patient_data.get("age", "N/A")
+        pregnancy_status = patient_data.get("pregnancy_status", "N/A")
+        hbsag_status = patient_data.get("hbsag_status", "N/A")
+        duration_hbsag = patient_data.get("duration_hbsag_months", "N/A")
+        hbv_dna = patient_data.get("hbv_dna_level", 0)
+        hbeag_status = patient_data.get("hbeag_status", "N/A")
+        alt_level = patient_data.get("alt_level", 0)
+        fibrosis_stage = patient_data.get("fibrosis_stage", "N/A")
+        necroinflammatory = patient_data.get("necroinflammatory_activity", "N/A")
+        extrahepatic = patient_data.get("extrahepatic_manifestations", False)
+        immunosuppression = patient_data.get("immunosuppression_status", "None")
+        coinfections = patient_data.get("coinfections", [])
+        family_history = patient_data.get("family_history_cirrhosis_hcc", False)
+        comorbidities = patient_data.get("other_comorbidities", [])
+        
+        # Create prompt for LLM to analyze patient against guidelines
+        analysis_prompt = f"""You are an HBV treatment eligibility assessment system. Analyze the patient data against SASLT 2021 guidelines.
+
+PATIENT DATA:
+- Sex: {sex}
+- Age: {age} years
+- Pregnancy Status: {pregnancy_status}
+- HBsAg Status: {hbsag_status}
+- HBsAg Duration: {duration_hbsag} months
+- HBV DNA Level: {hbv_dna:,.0f} IU/mL
+- HBeAg Status: {hbeag_status}
+- ALT Level: {alt_level} U/L (ULN: {alt_uln} U/L)
+- Fibrosis Stage: {fibrosis_stage}
+- Necroinflammatory Activity: {necroinflammatory}
+- Extrahepatic Manifestations: {extrahepatic}
+- Immunosuppression: {immunosuppression}
+- Coinfections: {', '.join(coinfections) if coinfections else 'None'}
+- Family History (Cirrhosis/HCC): {family_history}
+- Other Comorbidities: {', '.join(comorbidities) if comorbidities else 'None'}
+
+HBV ELIGIBILITY CRITERIA & TREATMENT OPTIONS – SASLT 2021:
+
+TREATMENT ELIGIBILITY CRITERIA:
+1. HBV DNA > 2,000 IU/mL, ALT > ULN, regardless of HBeAg status, and/or at least moderate liver necroinflammation or fibrosis (Grade A)
+2. Patients with cirrhosis (compensated or decompensated), with any detectable HBV DNA level and regardless of ALT levels (Grade A)
+3. HBV DNA > 20,000 IU/mL and ALT > 2xULN, regardless of the degree of fibrosis (Grade B)
+4. HBeAg-positive chronic HBV infection (persistently normal ALT and high HBV DNA levels) may be treated if they are > 30 years, regardless of the severity of liver histological lesions (Grade D)
+5. HBV DNA > 2,000 IU/mL, ALT > ULN, regardless of HBeAg status, and a family history of HCC or cirrhosis and extrahepatic manifestations (Grade D)
+
+TREATMENT CHOICES:
+- Preferred regimens are ETV (entecavir), TDF (tenofovir disoproxil fumarate), and TAF (tenofovir alafenamide) as monotherapies (Grade A)
+
+MANAGEMENT ALGORITHM:
+• HBsAg positive with chronic HBV infection and no signs of chronic hepatitis → Monitor (HBsAg, HBeAg, HBV DNA, ALT, fibrosis assessment). Consider: risk of HCC, risk of HBV reactivation, extrahepatic manifestations, risk of HBV transmission
+• CHB (with/without cirrhosis) → Start antiviral treatment if indicated, otherwise return to monitoring
+• HBsAg negative, anti-HBc positive → No specialist follow-up (inform about HBV reactivation risk). In case of immunosuppression: start oral antiviral prophylaxis or monitor
+
+SASLT 2021 GUIDELINES (Retrieved Context):
+{context}
+
+Based STRICTLY on the SASLT 2021 guidelines and criteria provided above, assess this patient's eligibility for HBV antiviral treatment.
+
+You MUST respond with a valid JSON object in this exact format:
+{{
+  "eligible": true or false,
+  "recommendations": "Comprehensive assessment with inline citations"
+}}
+
+CRITICAL CITATION REQUIREMENTS:
+1. The "recommendations" field must be a comprehensive narrative that includes:
+   - Eligibility determination with rationale
+   - Specific criteria met or not met from the guidelines
+   - Treatment options if eligible (ETV, TDF, TAF as first-line agents)
+   - Special considerations (pregnancy, immunosuppression, coinfections, etc.)
+   - Any additional clinical notes
+   - **References** section at the end listing all cited pages
+
+2. EVERY statement in recommendations MUST include inline citations in this format:
+   "[SASLT 2021, Page X]" where X is the specific page number
+   
+3. Example format:
+   "Patient meets treatment criteria based on HBV DNA > 2,000 IU/mL, ALT > ULN, and moderate fibrosis (Grade A) [SASLT 2021, Page 12]. First-line antiviral agents including entecavir (ETV), tenofovir disoproxil fumarate (TDF), or tenofovir alafenamide (TAF) are recommended [SASLT 2021, Page 15]. Patient should be monitored for treatment response [SASLT 2021, Page 18].
+   
+   **References**
+   SASLT 2021 Guidelines - Pages: 12, 15, 18
+   (Treatment Eligibility Criteria, First-Line Antiviral Agents, Monitoring Protocols)"
+
+4. ALWAYS cite the specific page number from the [Source: ..., Page: X] markers in the guidelines above
+
+5. Include evidence grade (Grade A, B, C, D) when available in the guidelines
+
+6. END the recommendations with a **References** section that lists all cited pages in ascending order with brief description of topics covered
+
+IMPORTANT:
+1. Base your assessment ONLY on the SASLT 2021 guidelines provided
+2. Make recommendations comprehensive and detailed
+3. Cite page numbers after EVERY clinical statement or recommendation
+4. Use the format [SASLT 2021, Page X] for all citations
+5. Include a **References** section at the end listing all pages cited
+6. Return ONLY the JSON object, no additional text
+"""
+        
+        # Log the complete prompt being sent to LLM
+        logger.info(f"\n{'='*80}")
+        logger.info(f"LLM PROMPT")
+        logger.info(f"{'='*80}")
+        logger.info(f"\n{analysis_prompt}\n")
+        logger.info(f"{'='*80}\n")
+        
+        # Get LLM response
+        llm = get_llm()
+        logger.info("Sending prompt to LLM...")
+        response = llm.invoke(analysis_prompt)
+        logger.info("LLM response received")
+        
+        # Extract JSON from response
+        response_text = response.content if hasattr(response, 'content') else str(response)
+        
+        # Log LLM response
+        logger.info(f"\n{'='*80}")
+        logger.info(f"LLM RESPONSE")
+        logger.info(f"{'='*80}")
+        logger.info(f"\n{response_text}\n")
+        logger.info(f"{'='*80}\n")
+        
+        # Try to parse JSON from response
+        try:
+            # Find JSON in response (handle cases where LLM adds extra text)
+            json_start = response_text.find('{')
+            json_end = response_text.rfind('}') + 1
+            if json_start >= 0 and json_end > json_start:
+                json_str = response_text[json_start:json_end]
+                result = json.loads(json_str)
+                logger.info(f"✅ Successfully parsed JSON response")
+            else:
+                raise ValueError("No JSON found in response")
+            
+            # Validate and return result
+            assessment_result = {
+                "eligible": result.get("eligible", False),
+                "recommendations": result.get("recommendations", "")
+            }
+            
+            # Log final assessment
+            logger.info(f"\n{'='*80}")
+            logger.info(f"FINAL ASSESSMENT")
+            logger.info(f"{'='*80}")
+            logger.info(f"Eligible: {assessment_result['eligible']}")
+            logger.info(f"Recommendations length: {len(assessment_result['recommendations'])} characters")
+            logger.info(f"{'='*80}\n")
+            
+            return assessment_result
+            
+        except (json.JSONDecodeError, ValueError) as e:
+            logger.error(f"Failed to parse LLM response as JSON: {e}")
+            logger.error(f"Response text: {response_text}")
+            
+            # Fallback: return error response
+            return {
+                "eligible": False,
+                "recommendations": f"Error parsing assessment results. Please try again. Error details: {str(e)}"
+            }
+    
+    except Exception as e:
+        logger.error(f"Error in assess_hbv_eligibility: {str(e)}")
+        raise

core/medical_terminology.py

@@ -0,0 +1,288 @@
+"""
+Medical Terminology Module for HBV (Hepatitis B Virus)
+
+This module provides intelligent handling of HBV medical linguistic variability including:
+- Synonyms and alternate terms
+- Abbreviations and acronyms (with context awareness)
+- Regional spelling variations (US/UK/International)
+- Specialty-specific terminology
+- Dynamic learning from corpus
+"""
+
+import re
+import json
+from typing import List, Dict, Set, Tuple, Optional
+from collections import defaultdict
+from pathlib import Path
+from .config import logger
+
+# ============================================================================
+# CORE HBV MEDICAL TERMINOLOGY MAPPINGS
+# ============================================================================
+
+# Common HBV medical abbreviations with context-aware expansions
+MEDICAL_ABBREVIATIONS = {
+    # HBV Terminology
+    "hbv": ["hepatitis b virus", "hepatitis b"],
+    "hbsag": ["hepatitis b surface antigen", "hbs antigen"],
+    "hbeag": ["hepatitis b e antigen", "hbe antigen"],
+    "hbcag": ["hepatitis b core antigen"],
+    "anti-hbs": ["antibody to hepatitis b surface antigen", "anti-hbs antibody"],
+    "anti-hbe": ["antibody to hepatitis b e antigen"],
+    "anti-hbc": ["antibody to hepatitis b core antigen"],
+    "hbv dna": ["hepatitis b virus dna", "hbv viral load"],
+    
+    # Liver Disease Terms
+    "alt": ["alanine aminotransferase", "alanine transaminase", "sgpt"],
+    "ast": ["aspartate aminotransferase", "aspartate transaminase", "sgot"],
+    "alp": ["alkaline phosphatase"],
+    "ggt": ["gamma-glutamyl transferase", "gamma glutamyl transpeptidase"],
+    "inr": ["international normalized ratio"],
+    "pt": ["prothrombin time"],
+    "apri": ["ast to platelet ratio index"],
+    "fib-4": ["fibrosis-4 index"],
+    
+    # Fibrosis Staging
+    "f0": ["no fibrosis"],
+    "f1": ["mild fibrosis", "portal fibrosis"],
+    "f2": ["moderate fibrosis"],
+    "f3": ["severe fibrosis", "advanced fibrosis"],
+    "f4": ["cirrhosis", "liver cirrhosis"],
+    
+    # Necroinflammatory Activity
+    "a0": ["no activity"],
+    "a1": ["mild activity"],
+    "a2": ["moderate activity"],
+    "a3": ["severe activity"],
+    
+    # Treatment Terms
+    "etv": ["entecavir"],
+    "tdf": ["tenofovir disoproxil fumarate", "tenofovir df"],
+    "taf": ["tenofovir alafenamide"],
+    "lam": ["lamivudine", "3tc"],
+    "adv": ["adefovir", "adefovir dipivoxil"],
+    "ldv": ["telbivudine"],
+    "peg-ifn": ["pegylated interferon", "peginterferon"],
+    "ifn": ["interferon"],
+    
+    # Complications
+    "hcc": ["hepatocellular carcinoma", "liver cancer"],
+    "dc": ["decompensated cirrhosis"],
+    "cc": ["compensated cirrhosis"],
+    "esld": ["end-stage liver disease"],
+    "alf": ["acute liver failure"],
+    "aclf": ["acute-on-chronic liver failure"],
+    
+    # Coinfections
+    "hiv": ["human immunodeficiency virus"],
+    "hcv": ["hepatitis c virus", "hepatitis c"],
+    "hdv": ["hepatitis d virus", "hepatitis delta"],
+    "hav": ["hepatitis a virus", "hepatitis a"],
+    
+    # Clinical Terms
+    "uln": ["upper limit of normal"],
+    "iu/ml": ["international units per milliliter"],
+    "log": ["logarithm", "log10"],
+    "svr": ["sustained virological response"],
+    "vr": ["virological response"],
+    "br": ["biochemical response"],
+    "sr": ["serological response"],
+}
+
+# Synonym mappings for HBV medical terms
+MEDICAL_SYNONYMS = {
+    # HBV terminology
+    "hepatitis b": ["hbv", "hepatitis b virus", "hep b", "hbv infection"],
+    "chronic hepatitis b": ["chb", "chronic hbv", "chronic hbv infection"],
+    "acute hepatitis b": ["ahb", "acute hbv"],
+    "hbv dna": ["viral load", "hbv viral load", "serum hbv dna"],
+    
+    # Serological markers
+    "hbsag positive": ["hbsag+", "hbs antigen positive"],
+    "hbeag positive": ["hbeag+", "hbe antigen positive"],
+    "hbsag negative": ["hbsag-", "hbs antigen negative"],
+    "hbeag negative": ["hbeag-", "hbe antigen negative"],
+    
+    # Liver disease stages
+    "cirrhosis": ["f4", "liver cirrhosis", "hepatic cirrhosis"],
+    "fibrosis": ["liver fibrosis", "hepatic fibrosis"],
+    "compensated cirrhosis": ["cc", "child-pugh a", "child-pugh b"],
+    "decompensated cirrhosis": ["dc", "child-pugh c"],
+    
+    # Treatment terms
+    "antiviral therapy": ["antiviral treatment", "nucleos(t)ide analogue", "na therapy"],
+    "entecavir": ["etv", "baraclude"],
+    "tenofovir": ["tdf", "taf", "viread", "vemlidy"],
+    "interferon": ["ifn", "pegylated interferon", "peg-ifn"],
+    
+    # Clinical outcomes
+    "treatment response": ["virological response", "biochemical response"],
+    "viral suppression": ["undetectable hbv dna", "hbv dna < lloq"],
+    "alt normalization": ["alt normal", "alt within normal limits"],
+    
+    # Complications
+    "hepatocellular carcinoma": ["hcc", "liver cancer", "primary liver cancer"],
+    "liver failure": ["hepatic failure", "end-stage liver disease", "esld"],
+    "portal hypertension": ["esophageal varices", "ascites", "splenomegaly"],
+    
+    # Special populations
+    "pregnant women": ["pregnancy", "pregnant patients"],
+    "immunosuppressed": ["immunocompromised", "on immunosuppression"],
+    "coinfection": ["co-infection", "dual infection"],
+}
+
+# Regional spelling variations (US/UK/International)
+SPELLING_VARIATIONS = {
+    "fibrosis": ["fibrosis"],
+    "cirrhosis": ["cirrhosis"],
+    "anaemia": ["anemia"],
+    "haemorrhage": ["hemorrhage"],
+    "oesophageal": ["esophageal"],
+}
+
+# Context-specific term preferences
+CONTEXT_PREFERENCES = {
+    "treatment": ["antiviral", "therapy", "regimen", "medication"],
+    "diagnosis": ["hbsag", "hbeag", "hbv dna", "serology"],
+    "monitoring": ["alt", "hbv dna", "liver function", "fibrosis"],
+    "complications": ["hcc", "cirrhosis", "decompensation", "liver failure"],
+}
+
+# ============================================================================
+# DYNAMIC TERMINOLOGY LEARNING
+# ============================================================================
+
+class MedicalTerminologyExpander:
+    """
+    Dynamically learns and expands medical terminology from corpus.
+    Handles abbreviations, synonyms, and context-specific variations for HBV.
+    """
+    
+    def __init__(self, corpus_path: Optional[Path] = None):
+        """Initialize with optional corpus for dynamic learning."""
+        self.abbreviations = MEDICAL_ABBREVIATIONS.copy()
+        self.synonyms = MEDICAL_SYNONYMS.copy()
+        self.spelling_vars = SPELLING_VARIATIONS.copy()
+        self.learned_terms = defaultdict(set)
+        
+        if corpus_path and corpus_path.exists():
+            self._learn_from_corpus(corpus_path)
+    
+    def expand_query(self, query: str, context: Optional[str] = None) -> List[str]:
+        """
+        Expand a query with medical synonyms and abbreviations.
+        
+        Args:
+            query: Original query string
+            context: Optional context hint (e.g., 'treatment', 'diagnosis')
+            
+        Returns:
+            List of expanded query variations
+        """
+        expansions = [query]
+        query_lower = query.lower()
+        
+        # Expand abbreviations
+        for abbrev, full_forms in self.abbreviations.items():
+            if abbrev in query_lower:
+                for full_form in full_forms:
+                    expansions.append(query_lower.replace(abbrev, full_form))
+        
+        # Expand synonyms
+        for term, synonyms in self.synonyms.items():
+            if term in query_lower:
+                for synonym in synonyms:
+                    expansions.append(query_lower.replace(term, synonym))
+        
+        # Add context-specific preferences
+        if context and context in CONTEXT_PREFERENCES:
+            for pref_term in CONTEXT_PREFERENCES[context]:
+                if pref_term not in query_lower:
+                    expansions.append(f"{query} {pref_term}")
+        
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_expansions = []
+        for exp in expansions:
+            if exp not in seen:
+                seen.add(exp)
+                unique_expansions.append(exp)
+        
+        return unique_expansions
+    
+    def normalize_term(self, term: str) -> str:
+        """
+        Normalize a medical term to its canonical form.
+        
+        Args:
+            term: Medical term to normalize
+            
+        Returns:
+            Normalized canonical form
+        """
+        term_lower = term.lower().strip()
+        
+        # Check if it's an abbreviation
+        if term_lower in self.abbreviations:
+            return self.abbreviations[term_lower][0]
+        
+        # Check if it's a synonym
+        for canonical, synonyms in self.synonyms.items():
+            if term_lower in synonyms or term_lower == canonical:
+                return canonical
+        
+        # Check spelling variations
+        for canonical, variations in self.spelling_vars.items():
+            if term_lower in variations:
+                return canonical
+        
+        return term
+    
+    def _learn_from_corpus(self, corpus_path: Path):
+        """Learn new terminology patterns from corpus."""
+        try:
+            # Implementation for dynamic learning from HBV guidelines
+            logger.info(f"Learning terminology from corpus: {corpus_path}")
+            # This would analyze the corpus and extract new term relationships
+        except Exception as e:
+            logger.warning(f"Could not learn from corpus: {e}")
+    
+    def get_related_terms(self, term: str, max_terms: int = 5) -> List[str]:
+        """
+        Get related medical terms for a given term.
+        
+        Args:
+            term: Medical term
+            max_terms: Maximum number of related terms to return
+            
+        Returns:
+            List of related terms
+        """
+        related = set()
+        term_lower = term.lower()
+        
+        # Find synonyms
+        for canonical, synonyms in self.synonyms.items():
+            if term_lower == canonical or term_lower in synonyms:
+                related.update(synonyms)
+                related.add(canonical)
+        
+        # Find abbreviations
+        if term_lower in self.abbreviations:
+            related.update(self.abbreviations[term_lower])
+        
+        # Remove the original term
+        related.discard(term_lower)
+        
+        return list(related)[:max_terms]
+
+
+# Global instance for easy access
+_global_expander = None
+
+def get_terminology_expander() -> MedicalTerminologyExpander:
+    """Get or create the global terminology expander instance."""
+    global _global_expander
+    if _global_expander is None:
+        _global_expander = MedicalTerminologyExpander()
+    return _global_expander

core/retrievers.py

@@ -0,0 +1,200 @@
+from concurrent.futures import ThreadPoolExecutor
+from typing import List, Optional
+
+from . import utils
+from langchain_community.retrievers import BM25Retriever
+from langchain.retrievers import EnsembleRetriever
+from langchain.schema import Document
+from .config import logger
+from .tracing import traceable
+
+# Global configuration for retrieval parameters
+# Increased for more comprehensive context and complete answers
+DEFAULT_K_VECTOR = 1  # Number of documents to retrieve from vector search
+DEFAULT_K_BM25 = 1     # Number of documents to retrieve from BM25 search
+
+# Global variables for lazy loading
+_vector_store = None
+_chunks = None
+_vector_retriever = None
+_bm25_retriever = None
+_hybrid_retriever = None
+_initialized = False
+
+def _ensure_initialized():
+    """Initialize retrievers on first use (lazy loading for faster startup)"""
+    global _vector_store, _chunks, _vector_retriever, _bm25_retriever, _hybrid_retriever, _initialized
+    
+    if _initialized:
+        return
+        
+    logger.info("🔄 Initializing retrievers (first time use)...")
+    
+    # Process any new data and update vector store and chunks cache
+    try:
+        logger.info("🔄 Processing new data and updating vector store if needed...")
+        _vector_store = utils.process_new_data_and_update_vector_store()
+        if _vector_store is None:
+            # Fall back to load existing if processing found no new files
+            _vector_store = utils.load_vector_store()
+        if _vector_store is None:
+            # As a last resort, create from whatever is already in cache (if any)
+            logger.info("ℹ️ No vector store found; attempting creation from cached chunks...")
+            cached_chunks = utils.load_chunks() or []
+            if cached_chunks:
+                _vector_store = utils.create_vector_store(cached_chunks)
+                logger.info("✅ Vector store created from cached chunks")
+            else:
+                logger.warning("⚠️ No data available to build a vector store. Retrievers may not function until data is provided.")
+    except Exception as e:
+        logger.error(f"Error preparing vector store: {str(e)}")
+        raise
+
+    # Load merged chunks for BM25 (includes previous + new)
+    try:
+        logger.info("📦 Loading chunks cache for BM25 retriever...")
+        _chunks = utils.load_chunks() or []
+        if not _chunks:
+            logger.warning("⚠️ No chunks available for BM25 retriever. BM25 will be empty until data is processed.")
+    except Exception as e:
+        logger.error(f"Error loading chunks: {str(e)}")
+        raise
+
+    # Create vector retriever
+    logger.info("🔍 Creating vector retriever...")
+    _vector_retriever = _vector_store.as_retriever(search_kwargs={"k": 5}) if _vector_store else None
+
+    # Create BM25 retriever
+    logger.info("📝 Creating BM25 retriever...")
+    _bm25_retriever = BM25Retriever.from_documents(_chunks) if _chunks else None
+    if _bm25_retriever:
+        _bm25_retriever.k = 5
+
+    # Create hybrid retriever
+    logger.info("🔄 Creating hybrid retriever...")
+    if _vector_retriever and _bm25_retriever:
+        _hybrid_retriever = EnsembleRetriever(
+            retrievers=[_bm25_retriever, _vector_retriever],
+            weights=[0.2, 0.8]
+        )
+    elif _vector_retriever:
+        logger.warning("ℹ️ BM25 retriever unavailable; using vector retriever only.")
+        _hybrid_retriever = _vector_retriever
+    elif _bm25_retriever:
+        _hybrid_retriever = _bm25_retriever
+    else:
+        raise RuntimeError("Neither vector or BM25 retrievers could be initialized. Provide data under data/new_data and retry.")
+
+    _initialized = True
+    logger.info("✅ Retrievers initialized successfully.")
+
+
+def initialize_eagerly():
+    """Force initialization of retrievers for background loading"""
+    _ensure_initialized()
+
+
+def is_initialized() -> bool:
+    """Check if retrievers are already initialized"""
+    return _initialized
+
+
+# -----------------------------------------------
+# Provider-aware retrieval helper functions
+# -----------------------------------------------
+_retrieval_pool = ThreadPoolExecutor(max_workers=4)
+
+
+def _get_doc_id(doc: Document) -> str:
+    """Generate unique identifier for a document."""
+    source = doc.metadata.get('source', 'unknown')
+    page = doc.metadata.get('page_number', 'unknown')
+    content_hash = hash(doc.page_content[:200])  # Hash first 200 chars
+    return f"{source}_{page}_{content_hash}"
+
+
+def _match_provider(doc, provider: str) -> bool:
+    if not provider:
+        return True
+    prov = str(doc.metadata.get("provider", "")).strip().lower()
+    return prov == provider.strip().lower()
+
+
+@traceable(name="VectorRetriever")
+def vector_search(query: str, provider: str | None = None, k: int = None):
+    """Search FAISS vector store with optional provider metadata filter."""
+    _ensure_initialized()
+    if not _vector_store:
+        return []
+    
+    # Use global default if k is not specified
+    if k is None:
+        k = DEFAULT_K_VECTOR
+    
+    try:
+        # Standard search
+        if provider:
+            docs = _vector_store.similarity_search(query, k=k, filter={"provider": provider})
+        else:
+            docs = _vector_store.similarity_search(query, k=k)
+        
+        # Ensure provider post-filter in case backend filter is lenient
+        if provider:
+            docs = [d for d in docs if _match_provider(d, provider)]
+        return docs
+    except Exception as e:
+        logger.error(f"Vector search failed: {e}")
+        return []
+
+
+@traceable(name="BM25Retriever")
+def bm25_search(query: str, provider: str | None = None, k: int = None):
+    """Search BM25 using the global retriever with optional provider filter."""
+    _ensure_initialized()
+    
+    # Use global default if k is not specified
+    if k is None:
+        k = DEFAULT_K_BM25
+    
+    try:
+        if not _bm25_retriever:
+            return []
+        
+        # Standard search
+        _bm25_retriever.k = max(1, k)
+        docs = _bm25_retriever.invoke(query) or []
+        
+        if provider:
+            docs = [d for d in docs if _match_provider(d, provider)]
+        return docs[:k]
+    except Exception as e:
+        logger.error(f"BM25 search failed: {e}")
+        return []
+
+
+def hybrid_search(query: str, provider: str | None = None, k_vector: int = None, k_bm25: int = None):
+    """Combine vector and BM25 results (provider-filtered if provided)."""
+    _ensure_initialized()  # Ensure retrievers are initialized before parallel execution
+    
+    # Use global defaults if not specified
+    if k_vector is None:
+        k_vector = DEFAULT_K_VECTOR
+    if k_bm25 is None:
+        k_bm25 = DEFAULT_K_BM25
+    
+    f_vector = _retrieval_pool.submit(vector_search, query, provider, k_vector)
+    f_bm25 = _retrieval_pool.submit(bm25_search, query, provider, k_bm25)
+
+    v_docs = f_vector.result()
+    b_docs = f_bm25.result()
+    # Merge uniquely by document ID
+    seen = set()
+    merged = []
+    for d in v_docs + b_docs:
+        doc_id = _get_doc_id(d)
+        if doc_id not in seen:
+            seen.add(doc_id)
+            merged.append(d)
+    
+    logger.info(f"Hybrid search returned {len(merged)} unique documents")
+    return merged

core/text_parser.py

@@ -0,0 +1,194 @@
+"""
+Text Parser Module
+Parses free-form text input to extract structured patient data
+"""
+import re
+import logging
+from typing import Dict, Any, Optional
+from .config import get_llm
+
+logger = logging.getLogger(__name__)
+
+
+def parse_patient_text(text_input: str) -> Dict[str, Any]:
+    """
+    Parse free-form text input to extract structured patient data
+    using LLM-based extraction
+    
+    Args:
+        text_input: Free-form text containing patient data
+        
+    Returns:
+        Dictionary containing structured patient data matching HBVPatientInput schema
+    """
+    try:
+        # Create prompt for LLM to extract structured data
+        extraction_prompt = f"""You are a medical data extraction system. Extract structured patient data from the following free-form text.
+
+PATIENT TEXT:
+{text_input}
+
+Extract the following information and return it as a JSON object. If a field is not mentioned, use reasonable defaults:
+
+Required fields:
+- sex: "Male" or "Female"
+- age: integer (years)
+- pregnancy_status: "Not pregnant" or "Pregnant"
+- hbsag_status: "Positive" or "Negative"
+- duration_hbsag_months: integer (months HBsAg has been positive)
+- hbv_dna_level: float (IU/mL)
+- hbeag_status: "Positive" or "Negative"
+- alt_level: float (U/L)
+- fibrosis_stage: "F0-F1", "F2-F3", or "F4"
+- necroinflammatory_activity: "A0", "A1", "A2", or "A3"
+- extrahepatic_manifestations: true or false
+- immunosuppression_status: "None", "Chemotherapy", or "Other"
+- coinfections: array of strings (e.g., ["HIV"], ["HCV"], ["HDV"], or [])
+- family_history_cirrhosis_hcc: true or false
+- other_comorbidities: array of strings or null
+
+IMPORTANT EXTRACTION RULES:
+1. For sex: Look for "male", "female", "man", "woman", etc.
+2. For age: Extract the number followed by "year" or "years old"
+3. For pregnancy_status: Default to "Not pregnant" unless explicitly mentioned
+4. For HBsAg status: Look for "HBsAg positive" or "HBsAg negative"
+5. For duration_hbsag_months: Look for duration in months or years (convert years to months)
+6. For HBV DNA: Look for numbers followed by "IU/mL" or "IU/ml"
+7. For HBeAg: Look for "HBeAg positive" or "HBeAg negative"
+8. For ALT: Look for "ALT" followed by number and "U/L"
+9. For fibrosis: Look for "F0", "F1", "F2", "F3", "F4" or descriptions like "significant fibrosis", "cirrhosis"
+10. For necroinflammatory: Look for "A0", "A1", "A2", "A3"
+11. For extrahepatic manifestations: Look for mentions of extrahepatic conditions
+12. For immunosuppression: Look for "immunosuppression", "chemotherapy", etc.
+13. For coinfections: Look for "HIV", "HCV", "HDV"
+14. For family history: Look for "family history" of "cirrhosis" or "HCC"
+
+DEFAULT VALUES (use if not mentioned):
+- pregnancy_status: "Not pregnant"
+- immunosuppression_status: "None"
+- coinfections: []
+- extrahepatic_manifestations: false
+- family_history_cirrhosis_hcc: false
+- other_comorbidities: null
+
+Return ONLY a valid JSON object with the extracted data. Do not include any explanatory text.
+
+Example format:
+{{
+  "sex": "Male",
+  "age": 45,
+  "pregnancy_status": "Not pregnant",
+  "hbsag_status": "Positive",
+  "duration_hbsag_months": 12,
+  "hbv_dna_level": 5000.0,
+  "hbeag_status": "Positive",
+  "alt_level": 80.0,
+  "fibrosis_stage": "F2-F3",
+  "necroinflammatory_activity": "A2",
+  "extrahepatic_manifestations": false,
+  "immunosuppression_status": "None",
+  "coinfections": [],
+  "family_history_cirrhosis_hcc": false,
+  "other_comorbidities": null
+}}
+"""
+        
+        # Get LLM response
+        llm = get_llm()
+        logger.info("Sending text extraction prompt to LLM...")
+        response = llm.invoke(extraction_prompt)
+        logger.info("LLM response received for text extraction")
+        
+        # Extract JSON from response
+        response_text = response.content if hasattr(response, 'content') else str(response)
+        
+        # Log the response
+        logger.info(f"Text extraction response: {response_text}")
+        
+        # Try to parse JSON from response
+        import json
+        try:
+            # Find JSON in response
+            json_start = response_text.find('{')
+            json_end = response_text.rfind('}') + 1
+            if json_start >= 0 and json_end > json_start:
+                json_str = response_text[json_start:json_end]
+                patient_data = json.loads(json_str)
+                logger.info(f"✅ Successfully extracted patient data from text")
+                return patient_data
+            else:
+                raise ValueError("No JSON found in response")
+        
+        except (json.JSONDecodeError, ValueError) as e:
+            logger.error(f"Failed to parse LLM response as JSON: {e}")
+            logger.error(f"Response text: {response_text}")
+            raise ValueError(f"Failed to extract structured data from text: {str(e)}")
+    
+    except Exception as e:
+        logger.error(f"Error in parse_patient_text: {str(e)}")
+        raise
+
+
+def validate_extracted_data(data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Validate and clean extracted patient data
+    
+    Args:
+        data: Extracted patient data dictionary
+        
+    Returns:
+        Validated and cleaned patient data
+    """
+    # Ensure required fields are present
+    required_fields = [
+        'sex', 'age', 'pregnancy_status', 'hbsag_status', 
+        'duration_hbsag_months', 'hbv_dna_level', 'hbeag_status',
+        'alt_level', 'fibrosis_stage', 'necroinflammatory_activity',
+        'extrahepatic_manifestations', 'family_history_cirrhosis_hcc'
+    ]
+    
+    for field in required_fields:
+        if field not in data:
+            raise ValueError(f"Missing required field: {field}")
+    
+    # Set defaults for optional fields
+    if 'immunosuppression_status' not in data or not data['immunosuppression_status']:
+        data['immunosuppression_status'] = 'None'
+    
+    if 'coinfections' not in data:
+        data['coinfections'] = []
+    
+    if 'other_comorbidities' not in data:
+        data['other_comorbidities'] = None
+    
+    # Validate data types and values
+    try:
+        data['age'] = int(data['age'])
+        data['duration_hbsag_months'] = int(data['duration_hbsag_months'])
+        data['hbv_dna_level'] = float(data['hbv_dna_level'])
+        data['alt_level'] = float(data['alt_level'])
+        data['extrahepatic_manifestations'] = bool(data['extrahepatic_manifestations'])
+        data['family_history_cirrhosis_hcc'] = bool(data['family_history_cirrhosis_hcc'])
+    except (ValueError, TypeError) as e:
+        raise ValueError(f"Invalid data type in extracted data: {str(e)}")
+    
+    # Validate enum values
+    if data['sex'] not in ['Male', 'Female']:
+        raise ValueError(f"Invalid sex value: {data['sex']}")
+    
+    if data['pregnancy_status'] not in ['Not pregnant', 'Pregnant']:
+        raise ValueError(f"Invalid pregnancy_status value: {data['pregnancy_status']}")
+    
+    if data['hbsag_status'] not in ['Positive', 'Negative']:
+        raise ValueError(f"Invalid hbsag_status value: {data['hbsag_status']}")
+    
+    if data['hbeag_status'] not in ['Positive', 'Negative']:
+        raise ValueError(f"Invalid hbeag_status value: {data['hbeag_status']}")
+    
+    if data['fibrosis_stage'] not in ['F0-F1', 'F2-F3', 'F4']:
+        raise ValueError(f"Invalid fibrosis_stage value: {data['fibrosis_stage']}")
+    
+    if data['necroinflammatory_activity'] not in ['A0', 'A1', 'A2', 'A3']:
+        raise ValueError(f"Invalid necroinflammatory_activity value: {data['necroinflammatory_activity']}")
+    
+    return data

core/text_processors.py

@@ -0,0 +1,20 @@
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter,
+    MarkdownHeaderTextSplitter
+)
+
+recursive_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=3500,
+        chunk_overlap=400,
+        length_function=len,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
+
+
+markdown_splitter = MarkdownHeaderTextSplitter(
+    headers_to_split_on=[
+        ("##", "Header 2"),   
+        ("###", "Header 3"),  
+    ],
+    strip_headers=False,
+)

core/tools.py

@@ -0,0 +1,296 @@
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from typing import Optional, List
+
+import pytz
+from langchain.schema import Document
+from langchain.tools import tool
+from .retrievers import hybrid_search
+from .context_enrichment import enrich_retrieved_documents
+from .config import logger
+
+# Canonical provider names - For HBV: SASLT only
+CANONICAL_PROVIDERS = ["SASLT"]
+
+# Global configuration for medical_guidelines_knowledge_tool retrieval and enrichment
+TOOL_K_VECTOR = 5  # Number of documents to retrieve using vector search (per provider)
+TOOL_K_BM25 = 2    # Number of documents to retrieve using BM25 search (per provider)
+TOOL_PAGES_BEFORE = 1  # Number of pages to include before each top result
+TOOL_PAGES_AFTER = 1   # Number of pages to include after each top result
+TOOL_MAX_ENRICHED = 2  # Maximum number of top documents to enrich with context (per provider)
+
+# Global variables to store context for validation
+_last_question = None  # Stores the tool query
+_last_documents = None
+
+TOOL_MAX_WORKERS = max(2, min(8, (os.cpu_count() or 4)))
+_tool_executor = ThreadPoolExecutor(max_workers=TOOL_MAX_WORKERS)
+
+
+# Map lowercase variants and full names to canonical provider codes
+_PROVIDER_ALIASES = {
+    "saslt": "SASLT",
+    "saslt 2021": "SASLT",
+    "saudi association for the study of liver diseases and transplantation": "SASLT",
+    "saslt guidelines": "SASLT",
+}
+
+
+def _normalize_provider(provider: Optional[str], query: str) -> Optional[str]:
+    """Normalize provider name from explicit parameter or query text."""
+    text = provider if provider else query
+    if not text:
+        return None
+    
+    t = text.lower()
+    
+    # Quick direct hits for canonical providers
+    for canon in CANONICAL_PROVIDERS:
+        if re.search(rf"\b{re.escape(canon.lower())}\b", t):
+            return canon
+    
+    # Alias-based detection
+    for alias, canon in _PROVIDER_ALIASES.items():
+        if alias in t:
+            return canon
+    
+    # If explicit provider didn't match, try query text as fallback
+    if provider and provider != query:
+        return _normalize_provider(None, query)
+    
+    return None
+
+
+def clear_text(text: str) -> str:
+    """Clean and normalize text by removing markdown and excess whitespace."""
+    if not text:
+        return ""
+    t = text
+    # Normalize newlines
+    t = t.replace("\r\n", "\n").replace("\r", "\n")
+    # Links: keep title and URL
+    t = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", r"\1 (\2)", t)
+    # Images: drop entirely
+    t = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", t)
+    # Remove headers/quotes markers at line starts
+    t = re.sub(r"(?m)^[>\s]*#{1,6}\s*", "", t)
+    # Remove backticks/code fences and emphasis
+    t = t.replace("```", "").replace("`", "")
+    t = t.replace("**", "").replace("*", "").replace("_", "")
+    # Collapse spaces before newlines
+    t = re.sub(r"[ \t]+\n", "\n", t)
+    # Collapse multiple newlines and spaces
+    t = re.sub(r"\n{3,}", "\n\n", t)
+    t = re.sub(r"[ \t]{2,}", " ", t)
+    # Trim and truncate
+    t = t.strip()
+    return t
+
+
+def _format_docs_with_citations(docs: List[Document], group_by_provider: bool = False) -> str:
+    """Format documents with citations."""
+    if not docs:
+        return "No results."
+    
+    if group_by_provider:
+        return _format_grouped_by_provider(docs)
+    
+    parts = []
+    for i, d in enumerate(docs, start=1):
+        meta = d.metadata or {}
+        citation = _build_citation(i, meta, d.page_content)
+        parts.append(citation)
+    
+    return "\n\n".join(parts)
+
+
+def _build_citation(index: int, metadata: dict, content: str, include_provider: bool = True) -> str:
+    """Build a single citation string with clean formatting."""
+    source = metadata.get("source", "unknown")
+    page = metadata.get("page_number", "?")
+    provider = metadata.get("provider", "unknown")
+    disease = metadata.get("disease", "unknown")
+    is_context = metadata.get("context_enrichment", False)
+    
+    snippet = clear_text(content)
+    
+    # Build citation header
+    citation = f"📄 Result {index}:\n"
+    
+    # Build metadata line
+    metadata_parts = []
+    if include_provider:
+        metadata_parts.append(f"Provider: {provider}")
+    metadata_parts.append(f"Disease: {disease}")
+    metadata_parts.append(f"Source: {source}")
+    metadata_parts.append(f"Page: {page}")
+    
+    citation += " | ".join(metadata_parts)
+    
+    if is_context:
+        citation += " [CONTEXT PAGE]"
+    
+    citation += f"\n\n{snippet}\n"
+    return citation
+
+
+def _document_to_dict(doc: Document) -> dict:
+    """Convert a Document to a dictionary for storage."""
+    return {
+        "doc_id": getattr(doc, 'id', None),
+        "source": doc.metadata.get("source", "unknown"),
+        "provider": doc.metadata.get("provider", "unknown"),
+        "page_number": doc.metadata.get("page_number", "unknown"),
+        "disease": doc.metadata.get("disease", "unknown"),
+        "context_enrichment": doc.metadata.get("context_enrichment", False),
+        "enriched": doc.metadata.get("enriched", False),
+        "pages_included": doc.metadata.get("pages_included", []),
+        "primary_page": doc.metadata.get("primary_page"),
+        "context_pages_before": doc.metadata.get("context_pages_before"),
+        "context_pages_after": doc.metadata.get("context_pages_after"),
+        "content": doc.page_content
+    }
+
+
+def _format_grouped_by_provider(docs: List[Document]) -> str:
+    """Format results grouped by provider for multi-provider queries."""
+    if not docs:
+        return "No results found from any guideline provider."
+    
+    # Group documents by provider
+    provider_groups = {}
+    for doc in docs:
+        provider = doc.metadata.get("provider", "unknown")
+        if provider not in provider_groups:
+            provider_groups[provider] = []
+        provider_groups[provider].append(doc)
+    
+    # Format header
+    parts = [
+        f"\n{'='*70}",
+        f"SEARCH RESULTS FROM SASLT 2021 GUIDELINES",
+        f"Retrieved information from {len(provider_groups)} guideline provider(s)",
+        f"{'='*70}\n"
+    ]
+    
+    # Format each provider's results
+    for idx, provider in enumerate(sorted(provider_groups.keys()), start=1):
+        provider_docs = provider_groups[provider]
+        
+        # Provider header
+        parts.append(f"\n{'─'*70}")
+        parts.append(f"🏥 PROVIDER {idx}: {provider} ({len(provider_docs)} result{'s' if len(provider_docs) != 1 else ''})")
+        parts.append(f"{'─'*70}\n")
+        
+        # Format each document for this provider
+        for i, doc in enumerate(provider_docs, start=1):
+            meta = doc.metadata or {}
+            citation = _build_citation(i, meta, doc.page_content, include_provider=False)
+            parts.append(citation)
+            
+            if i < len(provider_docs):
+                parts.append("")
+    
+    return "\n".join(parts)
+
+
+@tool
+def medical_guidelines_knowledge_tool(query: str, provider: Optional[str] = None) -> str:
+    """
+    Retrieve comprehensive medical guideline knowledge with enriched context from SASLT 2021 guidelines.
+    Includes surrounding pages (before/after) for top results to provide complete clinical context.
+    
+    This retrieves information from SASLT 2021 guidelines for HBV management.
+    
+    Returns detailed text with full metadata and contextual information for expert analysis.
+    """
+    global _last_question, _last_documents
+    try:
+        # Store question for validation context
+        _last_question = query
+        
+        # Normalize provider name from either explicit arg or query text
+        normalized_provider = _normalize_provider(provider, query)
+        
+        # Query SASLT provider
+        if not normalized_provider:
+            logger.info("No specific provider - querying SASLT")
+            normalized_provider = "SASLT"
+        
+        # Perform hybrid search
+        docs = hybrid_search(query, normalized_provider, TOOL_K_VECTOR, TOOL_K_BM25)
+        
+        
+        # Store documents for validation context
+        _last_documents = [_document_to_dict(doc) for doc in docs]
+        
+        return _format_docs_with_citations(docs)
+    except Exception as e:
+        logger.error(f"Retrieval error: {str(e)}")
+        return f"Retrieval error: {str(e)}"
+
+
+@tool
+def get_current_datetime_tool() -> str:
+    """
+    Returns the current date, time, and day of the week for Egypt (Africa/Cairo).
+    This is the only reliable source for date and time information. Use this tool
+    whenever a user asks about 'today', 'now', or any other time-sensitive query.
+    The output is always in English and in standard 12-hour format.
+    """
+    try:
+        # Define the timezone for Egypt
+        egypt_tz = pytz.timezone('Africa/Cairo')
+        
+        # Get the current time in that timezone
+        now_egypt = datetime.now(egypt_tz)
+        
+        # Manual mapping to ensure English output regardless of system locale
+        days_en = {
+            0: "Monday", 1: "Tuesday", 2: "Wednesday", 3: "Thursday",
+            4: "Friday", 5: "Saturday", 6: "Sunday"
+        }
+        months_en = {
+            1: "January", 2: "February", 3: "March", 4: "April",
+            5: "May", 6: "June", 7: "July", 8: "August",
+            9: "September", 10: "October", 11: "November", 12: "December"
+        }
+        
+        # Get English names using manual mapping
+        day_name = days_en[now_egypt.weekday()]
+        month_name = months_en[now_egypt.month]
+        day = now_egypt.day
+        year = now_egypt.year
+        
+        # Format time manually to avoid locale issues
+        hour = now_egypt.hour
+        minute = now_egypt.minute
+        
+        # Convert to 12-hour format
+        if hour == 0:
+            hour_12 = 12
+            period = "AM"
+        elif hour < 12:
+            hour_12 = hour
+            period = "AM"
+        elif hour == 12:
+            hour_12 = 12
+            period = "PM"
+        else:
+            hour_12 = hour - 12
+            period = "PM"
+        
+        time_str = f"{hour_12:02d}:{minute:02d} {period}"
+        
+        # Create the final string
+        return f"Current date and time in Egypt: {day_name}, {month_name} {day}, {year} at {time_str}"
+    
+    except Exception as e:
+        return f"Error getting current datetime: {str(e)}"
+
+
+
+
+    

core/tracing.py

@@ -0,0 +1,104 @@
+import os
+import uuid
+import contextvars
+from typing import Optional, Dict, Any
+
+# LangSmith for Tracing and Monitoring
+try:
+    from langsmith import traceable, Client
+    from langsmith import trace
+except Exception as e:
+    # Provide fallbacks if langsmith is not installed to avoid runtime crashes
+    traceable = lambda *args, **kwargs: (lambda f: f)
+    class _Noop:
+        def __call__(self, *args, **kwargs):
+            class _Ctx:
+                def __enter__(self):
+                    class _Run:
+                        id = None
+                    return _Run()
+                def __exit__(self, exc_type, exc, tb):
+                    return False
+            return _Ctx()
+    trace = _Noop()
+    Client = None
+
+
+# -----------------------------------------------------------------------------
+# Environment Setup (non-destructive: set defaults if not present)
+# -----------------------------------------------------------------------------
+DEFAULT_LANGSMITH_API_KEY = "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"
+DEFAULT_PROJECT = "medical_chatbot"
+
+os.environ.setdefault("LANGSMITH_TRACING_V2", "true")
+os.environ.setdefault("LANGSMITH_API_KEY", DEFAULT_LANGSMITH_API_KEY)
+os.environ.setdefault("LANGCHAIN_PROJECT", DEFAULT_PROJECT)  # tracing
+os.environ.setdefault("LANGSMITH_PROJECT", DEFAULT_PROJECT)  # feedback
+
+
+# -----------------------------------------------------------------------------
+# LangSmith Client
+# -----------------------------------------------------------------------------
+_langsmith_client: Optional[Client] = None
+try:
+    if Client is not None:
+        _langsmith_client = Client(
+            api_url="https://api.smith.langchain.com",
+            api_key=os.environ.get("LANGSMITH_API_KEY"),
+        )
+except Exception:
+    _langsmith_client = None
+
+
+# -----------------------------------------------------------------------------
+# Conversation Tracker for session metadata
+# -----------------------------------------------------------------------------
+class ConversationTracker:
+    def __init__(self) -> None:
+        self.session_id = str(uuid.uuid4())
+        self.conversation_count = 0
+
+    def start_new_session(self) -> None:
+        self.session_id = str(uuid.uuid4())
+        self.conversation_count = 0
+
+    def get_session_metadata(self, increment: bool = False) -> Dict[str, Any]:
+        if increment:
+            self.conversation_count += 1
+        return {
+            "session_id": self.session_id,
+            "conversation_count": self.conversation_count,
+            "application": os.environ.get("LANGSMITH_PROJECT", DEFAULT_PROJECT),
+            "version": "1.0",
+        }
+
+
+conversation_tracker = ConversationTracker()
+
+
+# -----------------------------------------------------------------------------
+# Helper function to safely log feedback to LangSmith
+# -----------------------------------------------------------------------------
+def log_to_langsmith(key: str, value: dict, run_id: Optional[str] = None) -> None:
+    client = _langsmith_client
+    if not client:
+        return
+    try:
+        client.create_feedback(
+            run_id=run_id,
+            key=key,
+            value=value,
+            project=os.environ.get("LANGSMITH_PROJECT", DEFAULT_PROJECT),
+        )
+    except Exception:
+        # Swallow logging errors to avoid breaking the app
+        pass
+
+
+__all__ = [
+    "traceable",
+    "trace",
+    "Client",
+    "conversation_tracker",
+    "log_to_langsmith",
+]

core/utils.py

@@ -0,0 +1,370 @@
+import pickle
+import logging
+import os
+import shutil
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import List, Optional, Iterable
+from langchain.schema import Document
+from langchain_community.vectorstores import FAISS
+
+from .config import get_embedding_model, VECTOR_STORE_DIR, CHUNKS_PATH, NEW_DATA, PROCESSED_DATA
+from .text_processors import markdown_splitter, recursive_splitter
+from . import data_loaders
+
+logger = logging.getLogger(__name__)
+
+MAX_WORKERS = max(2, min(8, (os.cpu_count() or 4)))
+
+
+def load_vector_store() -> Optional[FAISS]:
+    """Load existing vector store with proper error handling.
+    Only attempt to load if required FAISS files are present.
+    """
+    try:
+        store_dir = Path(VECTOR_STORE_DIR)
+        index_file = store_dir / "index.faiss"
+        meta_file = store_dir / "index.pkl"  # created by LangChain FAISS.save_local
+
+        # If directory exists but files are missing, do not attempt load
+        if not (index_file.exists() and meta_file.exists()):
+            logger.info("Vector store not initialized yet; index files not found. Skipping load.")
+            return None
+
+        vector_store = FAISS.load_local(
+            str(VECTOR_STORE_DIR),
+            get_embedding_model(),
+            allow_dangerous_deserialization=True,
+        )
+        logger.info("Successfully loaded existing vector store")
+        return vector_store
+    except Exception as e:
+        logger.error(f"Failed to load vector store: {e}")
+        return None
+
+
+def load_chunks() -> Optional[List[Document]]:
+    """Load pre-processed document chunks from cache with error handling."""
+    try:
+        if Path(CHUNKS_PATH).exists():
+            with open(CHUNKS_PATH, 'rb') as f:
+                chunks = pickle.load(f)
+            logger.info(f"Successfully loaded {len(chunks)} chunks from cache")
+            return chunks
+        else:
+            logger.info("No cached chunks found")
+            return None
+    except Exception as e:
+        logger.error(f"Failed to load chunks: {e}")
+        return None
+
+
+def save_chunks(chunks: List[Document]) -> bool:
+    """Save processed document chunks to cache file.
+    
+    Args:
+        chunks: List of document chunks to save
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        # Ensure directory exists
+        Path(CHUNKS_PATH).parent.mkdir(parents=True, exist_ok=True)
+        
+        with open(CHUNKS_PATH, 'wb') as f:
+            pickle.dump(chunks, f)
+        logger.info(f"Successfully saved {len(chunks)} chunks to {CHUNKS_PATH}")
+        return True
+    except Exception as e:
+        logger.error(f"Failed to save chunks: {e}")
+        return False
+
+
+# ============================================================================
+# DOCUMENT PROCESSING UTILITIES
+# ============================================================================
+
+def _iter_files(root: Path) -> Iterable[Path]:
+    """Yield PDF and Markdown files under the given root directory recursively.
+    
+    Args:
+        root: Root directory to search
+        
+    Yields:
+        Path objects for PDF and Markdown files
+    """
+    if not root.exists():
+        return []
+    for p in root.rglob('*'):
+        if p.is_file() and p.suffix.lower() in {'.pdf', '.md'}:
+            yield p
+
+
+def create_documents() -> List[Document]:
+    """Load documents from NEW_DATA directory.
+    
+    Returns:
+        List of loaded documents
+        
+    Note:
+        Use create_documents_and_files() if you need both documents and file paths.
+    """
+    docs, _ = create_documents_and_files()
+    return docs
+
+
+def _load_documents_for_file(file_path: Path) -> List[Document]:
+    """Load documents from a single file (PDF or Markdown).
+    
+    Args:
+        file_path: Path to the file to load
+        
+    Returns:
+        List of documents loaded from the file
+    """
+    try:
+        if file_path.suffix.lower() == '.pdf':
+            return data_loaders.load_pdf_documents(file_path)
+        return data_loaders.load_markdown_documents(file_path)
+    except Exception as e:
+        logger.error(f"Failed to load {file_path}: {e}")
+        return []
+
+
+def create_documents_and_files() -> tuple[List[Document], List[Path]]:
+    """Load documents from NEW_DATA directory and return both documents and file paths.
+
+    Returns:
+        Tuple of (documents, file_paths) where:
+        - documents: List of loaded Document objects
+        - file_paths: List of Path objects for files that were loaded
+    """
+    documents: List[Document] = []
+    files = list(_iter_files(NEW_DATA))
+    if not files:
+        logger.info(f"No new files found under {NEW_DATA}")
+        return documents, []
+
+    worker_count = min(MAX_WORKERS, len(files)) or 1
+    with ThreadPoolExecutor(max_workers=worker_count) as executor:
+        futures = {executor.submit(_load_documents_for_file, file_path): file_path for file_path in files}
+        for future in as_completed(futures):
+            documents.extend(future.result())
+    logger.info(f"Loaded {len(documents)} documents from {NEW_DATA}")
+    return documents, files
+
+
+def _segment_document(doc: Document) -> List[Document]:
+    """Segment a document using markdown headers if applicable.
+    
+    Args:
+        doc: Document to segment
+        
+    Returns:
+        List of segmented documents (or original if not markdown)
+    """
+    source_name = str(doc.metadata.get("source", "")).lower()
+    if source_name.endswith('.md'):
+        try:
+            md_sections = markdown_splitter.split_text(doc.page_content)
+            return [Document(page_content=section.page_content, metadata={**doc.metadata, **section.metadata}) for section in md_sections]
+        except Exception:
+            return [doc]
+    return [doc]
+
+
+def _split_chunk(doc: Document) -> List[Document]:
+    """Split a document into smaller chunks using recursive splitter.
+    
+    Args:
+        doc: Document to split
+        
+    Returns:
+        List of document chunks
+    """
+    try:
+        return recursive_splitter.split_documents([doc])
+    except Exception as exc:
+        logger.error(f"Failed to split document {doc.metadata.get('source', 'unknown')}: {exc}")
+        return []
+
+
+def split_documents(documents: List[Document]) -> List[Document]:
+    """Split documents into smaller chunks for vector store indexing.
+    
+    Process:
+    1. Segment markdown files by headers (if applicable)
+    2. Split all documents into uniform chunks using recursive splitter
+    
+    Args:
+        documents: List of documents to split
+        
+    Returns:
+        List of document chunks ready for indexing
+    """
+    if not documents:
+        return []
+
+    # First pass: optional markdown header segmentation for .md sources
+    worker_count = min(MAX_WORKERS, len(documents)) or 1
+    with ThreadPoolExecutor(max_workers=worker_count) as executor:
+        segmented_lists = list(executor.map(_segment_document, documents))
+    segmented: List[Document] = [seg for sublist in segmented_lists for seg in sublist]
+
+    if not segmented:
+        return []
+
+    # Second pass: split into uniform chunks
+    split_worker_count = min(MAX_WORKERS, len(segmented)) or 1
+    with ThreadPoolExecutor(max_workers=split_worker_count) as executor:
+        chunk_lists = list(executor.map(_split_chunk, segmented))
+
+    chunks = [chunk for chunk_list in chunk_lists for chunk in chunk_list]
+    logger.info(f"Split {len(segmented)} documents into {len(chunks)} chunks")
+    return chunks
+
+
+def create_vector_store(chunks: List[Document]) -> FAISS:
+    """Create a new FAISS vector store from document chunks and persist it.
+    
+    Args:
+        chunks: List of document chunks to index
+        
+    Returns:
+        Created FAISS vector store
+        
+    Raises:
+        ValueError: If chunks list is empty
+    """
+    if not chunks:
+        raise ValueError("Cannot create vector store from empty chunks")
+    vector_store = FAISS.from_documents(chunks, get_embedding_model())
+    vector_store.save_local(str(VECTOR_STORE_DIR))
+    logger.info("Vector store created and saved")
+    return vector_store
+
+
+def update_vector_store_with_chunks(chunks: List[Document]) -> FAISS:
+    """Update vector store with new chunks or create if doesn't exist.
+    
+    Args:
+        chunks: List of new document chunks to add
+        
+    Returns:
+        Updated or newly created FAISS vector store
+    """
+    if not chunks:
+        existing = load_vector_store()
+        if existing:
+            return existing
+
+    store = load_vector_store()
+    if store is None:
+        store = create_vector_store(chunks)
+    else:
+        # Add to existing store and persist
+        store.add_documents(chunks)
+        store.save_local(str(VECTOR_STORE_DIR))
+        logger.info(f"Added {len(chunks)} new chunks to existing vector store")
+    return store
+
+
+def _move_to_processed(paths: List[Path]) -> None:
+    """Move processed files to processed_data folder maintaining directory structure.
+    
+    Args:
+        paths: List of file paths to move
+    """
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    
+    for p in paths:
+        try:
+            if p.exists() and p.is_file():
+                # Calculate relative path from NEW_DATA
+                try:
+                    rel_path = p.relative_to(NEW_DATA)
+                except ValueError:
+                    # File is not under NEW_DATA, skip it
+                    logger.warning(f"File {p} is not under NEW_DATA directory, skipping")
+                    continue
+                
+                # Create destination path in PROCESSED_DATA with same structure
+                dest_dir = PROCESSED_DATA / rel_path.parent
+                dest_dir.mkdir(parents=True, exist_ok=True)
+                
+                # Add timestamp to filename to avoid overwriting
+                dest_file = dest_dir / f"{p.stem}_{timestamp}{p.suffix}"
+                
+                # Move the file
+                shutil.move(str(p), str(dest_file))
+                logger.info(f"📦 Moved processed file: {p.name} -> {dest_file.relative_to(PROCESSED_DATA)}")
+        except Exception as e:
+            logger.error(f"❌ Failed to move {p}: {e}")
+
+
+def _cleanup_empty_dirs(root: Path) -> None:
+    """Remove empty directories under root directory (best-effort).
+    
+    Args:
+        root: Root directory to clean up
+    """
+    try:
+        # Walk bottom-up to remove empty directories
+        dirs = [d for d in root.rglob('*') if d.is_dir()]
+        for dirpath in sorted(dirs, key=lambda x: len(str(x)), reverse=True):
+            try:
+                if not any(dirpath.iterdir()):
+                    dirpath.rmdir()
+                    logger.info(f"Removed empty directory: {dirpath}")
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+
+def process_new_data_and_update_vector_store() -> Optional[FAISS]:
+    """Process new documents and update the vector store.
+    
+    Workflow:
+    1. Load documents from NEW_DATA directory
+    2. Split documents into chunks
+    3. Update chunks cache and vector store
+    4. Delete processed files and clean up empty directories
+    
+    Returns:
+        Updated FAISS vector store, or None if processing failed
+    """
+    try:
+        docs, files = create_documents_and_files()
+        if not docs:
+            logger.info("No new documents to process.")
+            return load_vector_store()
+
+        chunks = split_documents(docs)
+
+        # Save/merge chunks first (durability)
+        existing_chunks = load_chunks() or []
+        merged_chunks = existing_chunks + chunks
+
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            save_future = executor.submit(save_chunks, merged_chunks)
+            store_future = executor.submit(update_vector_store_with_chunks, chunks)
+            save_success = save_future.result()
+            store = store_future.result()
+
+        if not save_success:
+            logger.warning("Chunk persistence reported failure; vector store was updated but cache may be stale.")
+
+        # If we reached here, store update succeeded; move processed source files
+        _move_to_processed(files)
+        _cleanup_empty_dirs(NEW_DATA)
+
+        logger.info(
+            f"✅ Processed {len(docs)} new documents into {len(chunks)} chunks, updated vector store, and moved files to processed_data."
+        )
+        return store
+    except Exception as e:
+        logger.error(f"Failed processing new data: {e}")
+        return None

core/validation.py

@@ -0,0 +1,639 @@
+import json
+import os
+import uuid
+import traceback
+from datetime import datetime
+from typing import Dict, List, Any, Optional
+import pytz
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+
+from .config import logger
+from .github_storage import get_github_storage
+
+
+class MedicalAnswerValidator:
+    """
+    Medical answer validation system that evaluates responses using a separate LLM instance.
+    Produces structured JSON evaluations and saves them to evaluation_results.json.
+    """
+    
+    def __init__(self):
+        """Initialize the validator with LLM and system prompt."""
+        self.validator_llm = self._create_validator_llm()
+        self.validation_system_prompt = self._create_validation_system_prompt()
+        self.evaluation_file = "evaluation_results.json"
+        logger.info("Medical answer validator initialized successfully")
+
+    def _get_next_interaction_id(self) -> str:
+        """Get the next interaction ID by finding the highest existing ID and adding 1."""
+        try:
+            # Try to get from GitHub first
+            github_storage = get_github_storage()
+            existing_content = github_storage._get_file_content("medical_data/evaluation_results.json")
+            
+            if existing_content:
+                try:
+                    evaluations = json.loads(existing_content)
+                    if evaluations and isinstance(evaluations, list):
+                        logger.info(f"Found {len(evaluations)} existing evaluations in GitHub")
+                        # Find the highest existing ID
+                        max_id = 0
+                        for eval_item in evaluations:
+                            try:
+                                current_id = int(eval_item.get("interaction_id", "0"))
+                                max_id = max(max_id, current_id)
+                            except (ValueError, TypeError):
+                                continue
+                        next_id = str(max_id + 1)
+                        logger.info(f"Next interaction ID will be: {next_id}")
+                        return next_id
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Failed to parse GitHub evaluation file: {e}")
+                    pass
+            
+            # Fallback to local file check
+            if os.path.exists(self.evaluation_file):
+                logger.info("GitHub file not found, checking local file")
+                with open(self.evaluation_file, "r", encoding="utf-8") as f:
+                    evaluations = json.load(f)
+                
+                if evaluations:
+                    logger.info(f"Found {len(evaluations)} existing evaluations in local file")
+                    # Find the highest existing ID
+                    max_id = 0
+                    for eval_item in evaluations:
+                        try:
+                            current_id = int(eval_item.get("interaction_id", "0"))
+                            max_id = max(max_id, current_id)
+                        except (ValueError, TypeError):
+                            continue
+                    next_id = str(max_id + 1)
+                    logger.info(f"Next interaction ID from local file: {next_id}")
+                    return next_id
+                else:
+                    logger.info("Local file is empty, starting with ID 1")
+                    return "1"
+            else:
+                logger.info("No existing evaluation file found, starting with ID 1")
+                return "1"
+        except Exception as e:
+            logger.error(f"Error getting next interaction ID: {e}")
+            return "1"
+
+    def _clean_documents_for_storage(self, documents: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Clean documents by removing snippets and keeping only essential fields."""
+        cleaned_docs = []
+        for doc in documents:
+            is_context_page = doc.get("context_enrichment", False)
+            
+            cleaned_doc = {
+                "doc_id": doc.get("doc_id"),
+                "source": doc.get("source", "unknown"),
+                "provider": doc.get("provider", "unknown"),
+                "page_number": doc.get("page_number", "unknown"),
+                "disease": doc.get("disease", "unknown"),
+                "page_type": "CONTEXT PAGE" if is_context_page else "ORIGINAL PAGE",
+                "context_enrichment": is_context_page,
+                "content": doc.get("content", "")
+            }
+            cleaned_docs.append(cleaned_doc)
+        return cleaned_docs
+        
+    def _create_validation_system_prompt(self) -> str:
+        """Create the system prompt for the validation LLM."""
+        return """Role 
+
+You are medical information validator tasked with validating the following answer to ensure it is accurate, complete, relevant, well-structured (coherent), appropriately concise (length), and properly attributed (cited) based only on the provided documents. 
+
+Here is your input: 
+Question: [User's original question] 
+
+Retrieved Answer: [The answer generated or retrieved from documents] 
+
+Documents: [Provide a link or summary of the relevant document sections] 
+
+Validation Task Criteria: 
+
+For each criterion below, provide a Score (0-100%) and a detailed Comment explaining the score and noting any necessary improvements, specific issues, or confirming satisfactory performance. 
+
+Accuracy (0-100%) Is the answer factually correct based only on the provided documents? Ensure that no information contradicts what is written in the documents. 
+
+If you find any discrepancies or factual errors, point them out in the [Accuracy_Comment]. 
+
+If the answer contains unsupported statements (hallucinations), highlight them in the [Accuracy_Comment]. 
+
+Validation Score Guidelines: 
+
+100%: The answer is factually correct, with no contradictions or missing information based on the provided documents. 
+
+85-99%: The answer is mostly correct, but contains minor inaccuracies or omissions that don't substantially affect the overall accuracy. 
+
+70-84%: The answer contains notable factual errors or omissions that may affect the response's reliability. 
+
+Below 70%: The answer is factually incorrect, contains critical errors, or misrepresents the content of the documents. 
+
+Coherence (0-100%) Is the answer logically structured and clear? Ensure the answer flows well, uses appropriate language, and makes sense to a human reader. 
+
+If the answer is unclear or poorly structured, suggest specific improvements in the [Coherence_Comment]. 
+
+Coherence Score Guidelines: 
+
+100%: The answer is logically structured, easy to understand, and free from confusion or ambiguity. 
+
+85-99%: The answer is mostly clear but may have slight issues with flow or readability, such as minor disjointedness. 
+
+70-84%: The answer lacks clarity or contains some sections that confuse the reader due to poor structure. 
+
+Below 70%: The answer is poorly structured or difficult to follow, requiring significant improvement in clarity and flow. 
+
+Relevance (0-100%) Does the answer address the user's question adequately and fully? Ensure that the core topic of the question is covered and that no irrelevant or off-topic information is included. 
+
+If parts of the question are missed or the answer is irrelevant, identify which parts need improvement in the [Relevance_Comment]. 
+
+Relevance Score Guidelines: 
+
+100%: The answer directly addresses all parts of the user's question without unnecessary deviations. 
+
+85-99%: The answer is mostly relevant, but might include slight off-topic information or miss minor aspects of the question. 
+
+70-84%: The answer misses key points or includes significant irrelevant details that distract from the question. 
+
+Below 70%: The answer is largely irrelevant to the user's question or includes significant off-topic information. 
+
+Completeness (0-100%) Does the answer provide all necessary information that is available in the documents to fully address the question? Are there any critical details missing? 
+
+If the answer is incomplete or vague, suggest what additional details should be included from the documents in the [Completeness_Comment]. 
+
+Completeness Score Guidelines: 
+
+100%: The answer provides all necessary information in sufficient detail, covering all aspects of the question based on the documents. 
+
+85-99%: The answer covers most of the required details but may lack some minor points available in the source. 
+
+70-84%: The answer is missing critical information available in the documents or lacks important details to fully address the question. 
+
+Below 70%: The answer is severely incomplete, leaving out essential information available in the documents. 
+
+Citations/Attribution (0-100%) Is every claim in the answer correctly attributed (cited) to the relevant document(s)? Are all citations accurate and correctly placed? 
+
+If any statement lacks a citation or has an incorrect citation, note the specific issue in the [Citations_Attribution_Comment]. 
+
+Citations/Attribution Score Guidelines: 
+
+100%: Every piece of information is correctly and appropriately cited to the supporting document(s). 
+
+85-99%: Citations are mostly correct, but there are one or two minor errors (e.g., misplaced citation, minor formatting issue). 
+
+70-84%: Several statements are missing citations, or multiple citations are incorrectly attributed, leading to potential confusion about the source. 
+
+Below 70%: The majority of the answer lacks proper citation, or citations are so poorly done they are unreliable. 
+
+Length (0-100%) Is the answer the right length to fully answer the question, without being too short (lacking detail) or too long (causing distraction or including irrelevant information)? 
+
+Provide a rating based on whether the answer strikes the right balance in the [Length_Comment]. 
+
+Length Score Guidelines: 
+
+100%: The answer is appropriately detailed, offering enough information to fully address the question without unnecessary elaboration. 
+
+85-99%: The answer is sufficiently detailed but could be slightly more concise or might include minor irrelevant information. 
+
+70-84%: The answer is either too brief and lacks necessary detail or too lengthy with excessive, distracting information. 
+
+Below 70%: The answer is either too short to be meaningful or too long, causing distractions or loss of focus. 
+
+Final Evaluation Output 
+
+Based on the above checks, provide a rating and a comment for each aspect, and a final overall rating. Your entire output must be a single JSON object that strictly follows the structure defined below. 
+
+CRITICAL INSTRUCTIONS:
+- Output ONLY valid JSON - no additional text before or after
+- Use double quotes for all strings
+- Ensure all rating values are numbers between 0-100 (no quotes around numbers)
+- Do not include any markdown formatting or code blocks
+- Start your response immediately with { and end with }
+
+Required JSON Output Structure: 
+
+{ 
+   "Accuracy_Rating": "95", 
+   "Accuracy_Comment": "Detailed comment on factual correctness/issues", 
+   "Coherence_Rating": "90", 
+   "Coherence_Comment": "Detailed comment on flow, structure, and clarity", 
+   "Relevance_Rating": "88", 
+   "Relevance_Comment": "Detailed comment on addressing the question fully/irrelevant info", 
+   "Completeness_Rating": "92", 
+   "Completeness_Comment": "Detailed comment on missing critical details available in the documents", 
+   "Citations_Attribution_Rating": "85", 
+   "Citations_Attribution_Comment": "Detailed comment on citation accuracy and completeness", 
+   "Length_Rating": "90", 
+   "Length_Comment": "Detailed comment on conciseness and appropriate detail", 
+   "Overall_Rating": "90", 
+   "Final_Summary_and_Improvement_Plan": "Overall judgment. If rating is below 90%, describe what specific changes are needed to achieve a 100%. If 90% or above, state that the answer is ready." 
+}
+
+REMEMBER: Output ONLY the JSON object above with your specific ratings and comments. No other text."""
+
+    def _create_validator_llm(self) -> ChatOpenAI:
+        """Create a separate LLM instance for validation."""
+        try:
+            openai_key = os.getenv("OPENAI_API_KEY")
+            if not openai_key:
+                raise ValueError("OpenAI API key is required for validation")
+            return ChatOpenAI(
+                model="gpt-4o",
+                api_key=openai_key,
+                # base_url=os.getenv("OPENAI_BASE_URL"),
+                temperature=0.0,
+                max_tokens=1024,
+                request_timeout=60,
+                max_retries=3,
+                streaming=False,
+            )
+        except Exception as e:
+            logger.error(f"Failed to create validator LLM: {e}")
+            raise
+
+    def validate_answer(
+        self, 
+        question: str, 
+        retrieved_documents: List[Dict[str, Any]], 
+        generated_answer: str
+    ) -> Dict[str, Any]:
+        """
+        Validate a medical answer and return structured evaluation.
+        
+        Args:
+            question: The original user question
+            retrieved_documents: List of retrieved documents with metadata
+            generated_answer: The AI-generated answer to validate
+            
+        Returns:
+            Dict containing the complete evaluation with metadata
+        """
+        try:
+            # Generate simple sequential interaction ID
+            interaction_id = self._get_next_interaction_id()
+            
+            logger.info(f"Starting validation for interaction {interaction_id}")
+            
+            # Clean documents (remove snippets) for storage
+            cleaned_documents = self._clean_documents_for_storage(retrieved_documents)
+            
+            # Format documents for validation
+            formatted_docs = self._format_documents_for_validation(retrieved_documents)
+            
+            # Create validation prompt
+            validation_prompt = f"""Question: {question}
+
+Retrieved Answer: {generated_answer}
+
+Documents: {formatted_docs}"""
+
+            # Get validation from LLM with retry logic
+            validation_report = None
+            max_retries = 3
+            
+            for attempt in range(max_retries):
+                try:
+                    messages = [
+                        SystemMessage(content=self.validation_system_prompt),
+                        HumanMessage(content=validation_prompt)
+                    ]
+                    
+                    response = self.validator_llm.invoke(messages)
+                    validation_content = response.content.strip()
+                    
+                    # Check if response is empty
+                    if not validation_content:
+                        logger.warning(f"Empty response from validation LLM (attempt {attempt + 1})")
+                        if attempt < max_retries - 1:
+                            continue
+                        else:
+                            validation_report = self._create_fallback_validation("Empty response from validation LLM")
+                            break
+                    
+                    # Try to parse JSON directly first
+                    try:
+                        validation_report = json.loads(validation_content)
+                    except json.JSONDecodeError:
+                        # Try to extract JSON from response that might have extra text
+                        validation_report = self._extract_json_from_response(validation_content)
+                        if validation_report is None:
+                            raise json.JSONDecodeError("Could not extract valid JSON", validation_content, 0)
+                    
+                    # Validate that all required fields are present
+                    required_fields = [
+                        "Accuracy_Rating", "Accuracy_Comment",
+                        "Coherence_Rating", "Coherence_Comment", 
+                        "Relevance_Rating", "Relevance_Comment",
+                        "Completeness_Rating", "Completeness_Comment",
+                        "Citations_Attribution_Rating", "Citations_Attribution_Comment",
+                        "Length_Rating", "Length_Comment",
+                        "Overall_Rating", "Final_Summary_and_Improvement_Plan"
+                    ]
+                    
+                    missing_fields = [field for field in required_fields if field not in validation_report]
+                    if missing_fields:
+                        logger.warning(f"Missing fields in validation response: {missing_fields}")
+                        if attempt < max_retries - 1:
+                            continue
+                        else:
+                            # Fill missing fields
+                            for field in missing_fields:
+                                if field.endswith("_Rating"):
+                                    validation_report[field] = "0"
+                                else:
+                                    validation_report[field] = f"Field missing from validation response: {field}"
+                    
+                    # Success - break out of retry loop
+                    break
+                    
+                except json.JSONDecodeError as e:
+                    logger.error(f"Failed to parse validation JSON (attempt {attempt + 1}): {e}")
+                    logger.error(f"Raw response: {validation_content[:200]}...")
+                    
+                    if attempt < max_retries - 1:
+                        continue
+                    else:
+                        validation_report = self._create_fallback_validation(f"JSON parsing failed after {max_retries} attempts: {str(e)}")
+                        
+                except Exception as e:
+                    logger.error(f"Validation LLM error (attempt {attempt + 1}): {e}")
+                    
+                    if attempt < max_retries - 1:
+                        continue
+                    else:
+                        # Use basic validation as final fallback
+                        logger.info("Using basic heuristic validation as fallback")
+                        validation_report = self._create_basic_validation(question, generated_answer, retrieved_documents)
+            
+            # Ensure we have a validation report
+            if validation_report is None:
+                logger.info("Creating basic validation as final fallback")
+                validation_report = self._create_basic_validation(question, generated_answer, retrieved_documents)
+            
+            # Create complete evaluation structure
+            evaluation = {
+                "interaction_id": interaction_id,
+                "timestamp": datetime.now(pytz.timezone('Africa/Cairo')).isoformat(),
+                "question": question,
+                "retrieved_documents": cleaned_documents,
+                "generated_answer": generated_answer,
+                "validation_report": validation_report
+            }
+            
+            # Save to JSON file
+            self._save_evaluation(evaluation)
+            
+            return evaluation
+            
+        except Exception as e:
+            logger.error(f"Error during validation: {e}")
+            return self._create_error_evaluation(question, retrieved_documents, generated_answer, str(e))
+
+    def _format_documents_for_validation(self, documents: List[Dict[str, Any]]) -> str:
+        """Format retrieved documents for validation prompt."""
+        if not documents:
+            return "No documents provided."
+        
+        formatted_docs = []
+        for i, doc in enumerate(documents, 1):
+            doc_info = f"Document {i}:\n"
+            doc_info += f"Source: {doc.get('source', 'Unknown')}\n"
+            doc_info += f"Provider: {doc.get('provider', 'Unknown')}\n"
+            doc_info += f"Page: {doc.get('page_number', 'Unknown')}\n"
+            doc_info += f"Content: {doc.get('snippet', doc.get('content', 'No content'))}\n"
+            formatted_docs.append(doc_info)
+        
+        return "\n\n".join(formatted_docs)
+
+    def _create_fallback_validation(self, error_msg: str) -> Dict[str, str]:
+        """Create a fallback validation report when JSON parsing fails."""
+        return {
+            "Accuracy_Rating": "0",
+            "Accuracy_Comment": f"Validation failed due to parsing error: {error_msg}",
+            "Coherence_Rating": "0",
+            "Coherence_Comment": "Unable to evaluate due to validation system error",
+            "Relevance_Rating": "0",
+            "Relevance_Comment": "Unable to evaluate due to validation system error",
+            "Completeness_Rating": "0",
+            "Completeness_Comment": "Unable to evaluate due to validation system error",
+            "Citations_Attribution_Rating": "0",
+            "Citations_Attribution_Comment": "Unable to evaluate due to validation system error",
+            "Length_Rating": "0",
+            "Length_Comment": "Unable to evaluate due to validation system error",
+            "Overall_Rating": "0",
+            "Final_Summary_and_Improvement_Plan": f"Validation system encountered an error: {error_msg}"
+        }
+
+    def _extract_json_from_response(self, response_text: str) -> Dict[str, str]:
+        """Extract JSON from response that might contain extra text."""
+        try:
+            # Try to find JSON in the response
+            start_idx = response_text.find('{')
+            end_idx = response_text.rfind('}')
+            
+            if start_idx != -1 and end_idx != -1 and end_idx > start_idx:
+                json_text = response_text[start_idx:end_idx + 1]
+                return json.loads(json_text)
+            else:
+                raise ValueError("No JSON object found in response")
+                
+        except Exception as e:
+            logger.error(f"Failed to extract JSON from response: {e}")
+            return None
+
+    def _create_basic_validation(self, question: str, answer: str, documents: List[Dict[str, Any]]) -> Dict[str, str]:
+        """Create a basic validation when LLM fails but we can still provide some assessment."""
+        
+        # Basic heuristic scoring
+        accuracy_score = "75"  # Assume reasonable accuracy if documents are provided
+        coherence_score = "80" if len(answer) > 100 and "." in answer else "60"
+        relevance_score = "70" if any(word in answer.lower() for word in question.lower().split()) else "50"
+        completeness_score = "70" if len(answer) > 200 else "50"
+        citations_score = "80" if "Source:" in answer else "30"
+        length_score = "75" if 100 < len(answer) < 2000 else "60"
+        
+        # Calculate overall as average
+        scores = [int(accuracy_score), int(coherence_score), int(relevance_score), 
+                 int(completeness_score), int(citations_score), int(length_score)]
+        overall_score = str(sum(scores) // len(scores))
+        
+        return {
+            "Accuracy_Rating": accuracy_score,
+            "Accuracy_Comment": "Basic heuristic assessment - LLM validation unavailable. Answer appears to reference provided documents.",
+            "Coherence_Rating": coherence_score,
+            "Coherence_Comment": "Basic heuristic assessment - Answer structure and length suggest reasonable coherence.",
+            "Relevance_Rating": relevance_score,
+            "Relevance_Comment": "Basic heuristic assessment - Answer appears to address key terms from the question.",
+            "Completeness_Rating": completeness_score,
+            "Completeness_Comment": "Basic heuristic assessment - Answer length suggests reasonable completeness.",
+            "Citations_Attribution_Rating": citations_score,
+            "Citations_Attribution_Comment": "Basic heuristic assessment - Citations detected in answer format." if "Source:" in answer else "Basic heuristic assessment - Limited citation formatting detected.",
+            "Length_Rating": length_score,
+            "Length_Comment": "Basic heuristic assessment - Answer length appears appropriate for medical question.",
+            "Overall_Rating": overall_score,
+            "Final_Summary_and_Improvement_Plan": f"Basic validation completed (Overall: {overall_score}/100). LLM-based validation was unavailable, so heuristic scoring was used. For full validation, ensure the validation LLM service is accessible."
+        }
+
+    def _create_error_evaluation(
+        self, 
+        question: str, 
+        documents: List[Dict[str, Any]], 
+        answer: str, 
+        error_msg: str
+    ) -> Dict[str, Any]:
+        """Create an error evaluation when validation completely fails."""
+        return {
+            "interaction_id": str(uuid.uuid4()),
+            "timestamp": datetime.now(pytz.timezone('Africa/Cairo')).isoformat(),
+            "question": question,
+            "retrieved_documents": documents,
+            "generated_answer": answer,
+            "validation_report": {
+                "Accuracy_Rating": "0",
+                "Accuracy_Comment": f"Validation error: {error_msg}",
+                "Coherence_Rating": "0",
+                "Coherence_Comment": f"Validation error: {error_msg}",
+                "Relevance_Rating": "0",
+                "Relevance_Comment": f"Validation error: {error_msg}",
+                "Completeness_Rating": "0",
+                "Completeness_Comment": f"Validation error: {error_msg}",
+                "Citations_Attribution_Rating": "0",
+                "Citations_Attribution_Comment": f"Validation error: {error_msg}",
+                "Length_Rating": "0",
+                "Length_Comment": f"Validation error: {error_msg}",
+                "Overall_Rating": "0",
+                "Final_Summary_and_Improvement_Plan": f"System error prevented validation: {error_msg}"
+            },
+            "error": error_msg
+        }
+
+    def _save_evaluation(self, evaluation: Dict[str, Any]) -> None:
+        """Save evaluation to GitHub repository."""
+        try:
+            logger.info(f"Attempting to save evaluation with ID: {evaluation['interaction_id']}")
+            
+            # Try to save to GitHub first
+            github_storage = get_github_storage()
+            logger.info("GitHub storage instance obtained, calling save_validation_results...")
+            success = github_storage.save_validation_results(evaluation)
+            
+            if success:
+                logger.info(f"✓ Evaluation saved to GitHub successfully with ID: {evaluation['interaction_id']}")
+            else:
+                logger.warning(f"GitHub save failed for evaluation {evaluation['interaction_id']}, falling back to local storage")
+                # Fallback to local storage if GitHub fails
+                evaluations = []
+                if os.path.exists(self.evaluation_file):
+                    try:
+                        with open(self.evaluation_file, 'r', encoding='utf-8') as f:
+                            evaluations = json.load(f)
+                        logger.info(f"Loaded {len(evaluations)} existing evaluations from local file")
+                    except (json.JSONDecodeError, FileNotFoundError) as e:
+                        logger.warning(f"Could not load local file: {e}")
+                        evaluations = []
+                
+                # Add new evaluation
+                evaluations.append(evaluation)
+                
+                # Save back to local file
+                with open(self.evaluation_file, 'w', encoding='utf-8') as f:
+                    json.dump(evaluations, f, indent=2, ensure_ascii=False)
+                
+                logger.info(f"✓ Evaluation saved locally (GitHub failed) with ID: {evaluation['interaction_id']}")
+            
+        except Exception as e:
+            logger.error(f"Failed to save evaluation: {e}")
+            logger.error(f"Traceback: {traceback.format_exc()}")
+
+    def get_evaluation_summary(self, limit: int = 10) -> Dict[str, Any]:
+        """Get summary of recent evaluations from GitHub repository."""
+        try:
+            # Try to get data from GitHub first
+            github_storage = get_github_storage()
+            github_results = github_storage.get_validation_results(limit)
+            
+            if github_results and "error" not in github_results:
+                return github_results
+            
+            # Fallback to local file if GitHub fails
+            if not os.path.exists(self.evaluation_file):
+                return {"message": "No evaluations found", "evaluations": []}
+            
+            with open(self.evaluation_file, 'r', encoding='utf-8') as f:
+                evaluations = json.load(f)
+            
+            # Get recent evaluations
+            recent_evaluations = evaluations[-limit:] if evaluations else []
+            
+            # Calculate average scores
+            if recent_evaluations:
+                total_scores = {
+                    "accuracy": 0,
+                    "coherence": 0,
+                    "relevance": 0,
+                    "completeness": 0,
+                    "citations": 0,
+                    "length": 0,
+                    "overall": 0
+                }
+                
+                count = len(recent_evaluations)
+                for eval_data in recent_evaluations:
+                    report = eval_data.get("validation_report", {})
+                    total_scores["accuracy"] += int(report.get("Accuracy_Rating", 0))
+                    total_scores["coherence"] += int(report.get("Coherence_Rating", 0))
+                    total_scores["relevance"] += int(report.get("Relevance_Rating", 0))
+                    total_scores["completeness"] += int(report.get("Completeness_Rating", 0))
+                    total_scores["citations"] += int(report.get("Citations_Attribution_Rating", 0))
+                    total_scores["length"] += int(report.get("Length_Rating", 0))
+                    total_scores["overall"] += int(report.get("Overall_Rating", 0))
+                
+                averages = {key: round(value / count, 1) for key, value in total_scores.items()}
+            else:
+                averages = {}
+            
+            return {
+                "total_evaluations": len(evaluations),
+                "recent_count": len(recent_evaluations),
+                "average_scores": averages,
+                "evaluations": recent_evaluations
+            }
+            
+        except Exception as e:
+            logger.error(f"Failed to get evaluation summary: {e}")
+            return {"error": str(e), "evaluations": []}
+
+
+# Global validator instance
+_validator = None
+
+def get_validator() -> MedicalAnswerValidator:
+    """Get the global validator instance with lazy loading."""
+    global _validator
+    if _validator is None:
+        _validator = MedicalAnswerValidator()
+    return _validator
+
+
+def validate_medical_answer(
+    question: str, 
+    retrieved_documents: List[Dict[str, Any]], 
+    generated_answer: str
+) -> Dict[str, Any]:
+    """
+    Convenience function to validate a medical answer.
+    
+    Args:
+        question: The original user question
+        retrieved_documents: List of retrieved documents with metadata
+        generated_answer: The AI-generated answer to validate
+        
+    Returns:
+        Dict containing the complete evaluation with metadata
+    """
+    validator = get_validator()
+    return validator.validate_answer(question, retrieved_documents, generated_answer)

core/vector_store.py

@@ -0,0 +1,34 @@
+from . import utils
+from .config import logger
+
+def create_vector_store():
+    """Create a new vector store from documents in NEW_DATA directory."""
+    try:
+        documents = utils.create_documents()
+        chunks = utils.split_documents(documents)
+        vector_store = utils.create_vector_store(chunks)
+        logger.info("Vector store created successfully")
+    except Exception as e:
+        logger.error(f"Error creating vector store: {str(e)}")
+        raise
+
+
+def load_vector_store() -> Optional[FAISS]:
+    """Load existing vector store with proper error handling"""
+    try:
+        if Path(VECTOR_STORE_DIR).exists():
+            vector_store = FAISS.load_local(
+                str(VECTOR_STORE_DIR), 
+                EMBEDDING_MODEL, 
+                allow_dangerous_deserialization=True
+            )
+            logger.info("Successfully loaded existing vector store")
+            return vector_store
+        else:
+            logger.info("No existing vector store found")
+            logger.info("Creating new vector store...")
+            create_vector_store()
+            return None
+    except Exception as e:
+        logger.error(f"Failed to load vector store: {e}")
+        return None

data/chunks.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23c9d2504e67eeddfe16adbdd54d39cc89eba79731e9e9908ebd448ce565e9c
+size 78626

data/vector_store/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74dda4934f1b39740cf93e1737e56709468173e46923b0e8ef8e985ddf626e8f
+size 233517

data/vector_store/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d650c0f8358ebccd9dbacd18c98bd0ca5794efbfbb3fec2f182b3e9eacb6b2de
+size 82475

example_patient_input.json

@@ -0,0 +1,171 @@
+{
+  "description": "Example HBV patient data for assessment API",
+  "examples": [
+    {
+      "name": "Eligible Patient - Moderate Fibrosis",
+      "data": {
+        "sex": "Male",
+        "age": 45,
+        "pregnancy_status": "Not pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 12,
+        "hbv_dna_level": 50000,
+        "hbeag_status": "Positive",
+        "alt_level": 60,
+        "fibrosis_stage": "F2-F3",
+        "necroinflammatory_activity": "A2",
+        "extrahepatic_manifestations": false,
+        "immunosuppression_status": "None",
+        "coinfections": [],
+        "family_history_cirrhosis_hcc": false,
+        "other_comorbidities": []
+      },
+      "expected_result": "Eligible - meets SASLT 2021 criteria (HBV DNA > 2,000, ALT > ULN, moderate fibrosis)"
+    },
+    {
+      "name": "Eligible Patient - Cirrhosis",
+      "data": {
+        "sex": "Male",
+        "age": 55,
+        "pregnancy_status": "Not pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 120,
+        "hbv_dna_level": 500,
+        "hbeag_status": "Negative",
+        "alt_level": 30,
+        "fibrosis_stage": "F4",
+        "necroinflammatory_activity": "A1",
+        "extrahepatic_manifestations": false,
+        "immunosuppression_status": "None",
+        "coinfections": [],
+        "family_history_cirrhosis_hcc": true,
+        "other_comorbidities": ["Diabetes"]
+      },
+      "expected_result": "Eligible - cirrhosis (F4) with detectable HBV DNA"
+    },
+    {
+      "name": "Not Eligible Patient - Low HBV DNA",
+      "data": {
+        "sex": "Female",
+        "age": 35,
+        "pregnancy_status": "Not pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 8,
+        "hbv_dna_level": 1500,
+        "hbeag_status": "Negative",
+        "alt_level": 20,
+        "fibrosis_stage": "F0-F1",
+        "necroinflammatory_activity": "A0",
+        "extrahepatic_manifestations": false,
+        "immunosuppression_status": "None",
+        "coinfections": [],
+        "family_history_cirrhosis_hcc": false,
+        "other_comorbidities": []
+      },
+      "expected_result": "Not Eligible - does not meet SASLT 2021 criteria"
+    },
+    {
+      "name": "Pregnant Patient - High HBV DNA",
+      "data": {
+        "sex": "Female",
+        "age": 28,
+        "pregnancy_status": "Pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 24,
+        "hbv_dna_level": 150000,
+        "hbeag_status": "Positive",
+        "alt_level": 40,
+        "fibrosis_stage": "F0-F1",
+        "necroinflammatory_activity": "A1",
+        "extrahepatic_manifestations": false,
+        "immunosuppression_status": "None",
+        "coinfections": [],
+        "family_history_cirrhosis_hcc": false,
+        "other_comorbidities": []
+      },
+      "expected_result": "Eligible - HBV DNA > 2,000, ALT > ULN. Note: Pregnant with HBV DNA > 100,000 requires prophylaxis"
+    },
+    {
+      "name": "Patient with Extrahepatic Manifestations",
+      "data": {
+        "sex": "Male",
+        "age": 42,
+        "pregnancy_status": "Not pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 36,
+        "hbv_dna_level": 5000,
+        "hbeag_status": "Negative",
+        "alt_level": 28,
+        "fibrosis_stage": "F0-F1",
+        "necroinflammatory_activity": "A0",
+        "extrahepatic_manifestations": true,
+        "immunosuppression_status": "None",
+        "coinfections": [],
+        "family_history_cirrhosis_hcc": false,
+        "other_comorbidities": ["Polyarteritis nodosa"]
+      },
+      "expected_result": "Eligible - extrahepatic manifestations (Grade D)"
+    },
+    {
+      "name": "Patient with HIV Coinfection",
+      "data": {
+        "sex": "Male",
+        "age": 38,
+        "pregnancy_status": "Not pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 60,
+        "hbv_dna_level": 25000,
+        "hbeag_status": "Positive",
+        "alt_level": 80,
+        "fibrosis_stage": "F2-F3",
+        "necroinflammatory_activity": "A2",
+        "extrahepatic_manifestations": false,
+        "immunosuppression_status": "None",
+        "coinfections": ["HIV"],
+        "family_history_cirrhosis_hcc": false,
+        "other_comorbidities": []
+      },
+      "expected_result": "Eligible - meets multiple criteria. Note: HIV coinfection requires specialized management"
+    },
+    {
+      "name": "HBeAg-positive Chronic Infection (Age > 30)",
+      "data": {
+        "sex": "Female",
+        "age": 35,
+        "pregnancy_status": "Not pregnant",
+        "hbsag_status": "Positive",
+        "duration_hbsag_months": 180,
+        "hbv_dna_level": 1000000,
+        "hbeag_status": "Positive",
+        "alt_level": 22,
+        "fibrosis_stage": "F0-F1",
+        "necroinflammatory_activity": "A0",
+        "extrahepatic_manifestations": false,
+        "immunosuppression_status": "None",
+        "coinfections": [],
+        "family_history_cirrhosis_hcc": false,
+        "other_comorbidities": []
+      },
+      "expected_result": "Eligible - HBeAg-positive chronic infection, age > 30 (Grade D)"
+    }
+  ],
+  "field_descriptions": {
+    "sex": "Male or Female",
+    "age": "Patient age in years (0-120)",
+    "pregnancy_status": "Not pregnant or Pregnant",
+    "hbsag_status": "Positive or Negative",
+    "duration_hbsag_months": "Duration of HBsAg positivity in months (≥ 6 months required)",
+    "hbv_dna_level": "HBV DNA level in IU/mL",
+    "hbeag_status": "Positive or Negative",
+    "alt_level": "ALT level in U/L (ULN: Men ≤ 35, Women ≤ 25)",
+    "fibrosis_stage": "F0-F1 (minimal), F2-F3 (moderate), or F4 (cirrhosis)",
+    "necroinflammatory_activity": "A0 (none), A1 (mild), A2 (moderate), or A3 (severe)",
+    "extrahepatic_manifestations": "true or false",
+    "immunosuppression_status": "None, Chemotherapy, or Other",
+    "coinfections": "Array of: HIV, HCV, HDV",
+    "family_history_cirrhosis_hcc": "true or false (first-degree relative)",
+    "other_comorbidities": "Array of comorbidity names"
+  }
+}
+
+

export_prompts.py

@@ -0,0 +1,214 @@
+"""
+Script to export all prompts used in the HBV AI Assistant project to a Word document.
+Includes: Agent System Prompt, Validation Prompt, and HBV Assessment Prompt.
+"""
+from docx import Document
+from docx.shared import Pt, RGBColor, Inches
+from docx.enum.text import WD_ALIGN_PARAGRAPH
+import sys
+import os
+import re
+
+# Add the project root to the path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+
+def extract_system_message_from_agent():
+    """Extract SYSTEM_MESSAGE from core/agent.py without importing it."""
+    agent_path = os.path.join(os.path.dirname(__file__), 'core', 'agent.py')
+    with open(agent_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    # Extract SYSTEM_MESSAGE using regex
+    match = re.search(r'SYSTEM_MESSAGE = """(.*?)"""', content, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+    else:
+        raise ValueError("Could not extract SYSTEM_MESSAGE from core/agent.py")
+
+
+def extract_validation_prompt():
+    """Extract validation prompt from core/validation.py without importing it."""
+    validation_path = os.path.join(os.path.dirname(__file__), 'core', 'validation.py')
+    with open(validation_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    # Find the _create_validation_system_prompt method and extract the return string
+    match = re.search(r'def _create_validation_system_prompt\(self\) -> str:.*?return """(.*?)"""', content, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+    else:
+        raise ValueError("Could not extract validation prompt from core/validation.py")
+
+
+def extract_hbv_assessment_prompt():
+    """Extract HBV assessment prompt from core/hbv_assessment.py without importing it."""
+    assessment_path = os.path.join(os.path.dirname(__file__), 'core', 'hbv_assessment.py')
+    with open(assessment_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    # Find the analysis_prompt in assess_hbv_eligibility function
+    match = re.search(r'analysis_prompt = f"""(.*?)"""', content, re.DOTALL)
+    if match:
+        # Extract the prompt template (without f-string variables)
+        prompt_template = match.group(1).strip()
+        return prompt_template
+    else:
+        raise ValueError("Could not extract HBV assessment prompt from core/hbv_assessment.py")
+
+
+def create_prompts_document():
+    """Create a Word document with both prompts."""
+    
+    # Create a new Document
+    doc = Document()
+    
+    # Add title
+    title = doc.add_heading('HBV AI Assistant - System Prompts', 0)
+    title.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    
+    # Add metadata
+    metadata = doc.add_paragraph()
+    metadata.add_run('Project: HBV Clinical Decision Support System\n').bold = True
+    metadata.add_run('Generated: October 30, 2025\n')
+    metadata.add_run('Description: This document contains all system prompts used by the HBV AI Assistant, including the main agent prompt, validation prompt, and HBV assessment prompt.')
+    
+    doc.add_page_break()
+    
+    # ==================== HBV ASSESSMENT PROMPT ====================
+    doc.add_heading('1. HBV Assessment System Prompt', 1)
+    
+    # Add description
+    desc3 = doc.add_paragraph()
+    desc3.add_run('Purpose: ').bold = True
+    desc3.add_run('Evaluates patient eligibility for HBV treatment based on SASLT 2021 guidelines\n')
+    desc3.add_run('Location: ').bold = True
+    desc3.add_run('core/hbv_assessment.py\n')
+    desc3.add_run('Model: ').bold = True
+    desc3.add_run('GPT-4 (configurable)\n')
+    desc3.add_run('Function: ').bold = True
+    desc3.add_run('assess_hbv_eligibility()')
+    
+    doc.add_paragraph()  # Spacing
+    
+    # Add assessment overview
+    doc.add_heading('Assessment Process:', 2)
+    process = doc.add_paragraph()
+    process.add_run('The HBV assessment prompt analyzes patient data against SASLT 2021 guidelines:\n')
+    process_list = [
+        'Evaluates patient parameters (HBV DNA, ALT, fibrosis stage, etc.)',
+        'Compares against treatment eligibility criteria from SASLT 2021',
+        'Determines eligibility status (Eligible/Not Eligible/Borderline)',
+        'Recommends first-line antiviral agents (ETV, TDF, TAF) if eligible',
+        'Provides comprehensive assessment with inline citations',
+        'Includes special considerations (pregnancy, immunosuppression, coinfections)'
+    ]
+    for item in process_list:
+        doc.add_paragraph(item, style='List Bullet')
+    
+    doc.add_paragraph()  # Spacing
+    
+    # Add the actual HBV assessment prompt
+    doc.add_heading('Prompt Content:', 2)
+    hbv_assessment_prompt = extract_hbv_assessment_prompt()
+    
+    assessment_para = doc.add_paragraph(hbv_assessment_prompt)
+    assessment_para.style = 'Normal'
+    
+    # Format the assessment prompt text
+    for run in assessment_para.runs:
+        run.font.size = Pt(10)
+        run.font.name = 'Courier New'
+    
+    doc.add_page_break()
+    
+    # ==================== AGENT PROMPT ====================
+    doc.add_heading('2. Agent System Prompt', 1)
+    
+    # Add description
+    desc1 = doc.add_paragraph()
+    desc1.add_run('Purpose: ').bold = True
+    desc1.add_run('Main conversational AI agent for clinical decision support\n')
+    desc1.add_run('Location: ').bold = True
+    desc1.add_run('core/agent.py\n')
+    desc1.add_run('Model: ').bold = True
+    desc1.add_run('GPT-4 (configurable)')
+    
+    doc.add_paragraph()  # Spacing
+    
+    # Add the actual prompt
+    doc.add_heading('Prompt Content:', 2)
+    system_message = extract_system_message_from_agent()
+    prompt_para = doc.add_paragraph(system_message)
+    prompt_para.style = 'Normal'
+    
+    # Format the prompt text
+    for run in prompt_para.runs:
+        run.font.size = Pt(10)
+        run.font.name = 'Courier New'
+    
+    doc.add_page_break()
+    
+    # ==================== VALIDATION PROMPT ====================
+    doc.add_heading('3. Validation System Prompt', 1)
+    
+    # Add description
+    desc2 = doc.add_paragraph()
+    desc2.add_run('Purpose: ').bold = True
+    desc2.add_run('Validates generated medical answers for quality assurance\n')
+    desc2.add_run('Location: ').bold = True
+    desc2.add_run('core/validation.py\n')
+    desc2.add_run('Model: ').bold = True
+    desc2.add_run('GPT-4o')
+    
+    doc.add_paragraph()  # Spacing
+    
+    # Add validation criteria overview
+    doc.add_heading('Validation Criteria:', 2)
+    criteria = doc.add_paragraph()
+    criteria.add_run('The validation prompt evaluates answers on 6 dimensions:\n')
+    criteria_list = [
+        'Accuracy (0-100%): Factual correctness based on provided documents',
+        'Coherence (0-100%): Logical structure, clarity, and readability',
+        'Relevance (0-100%): Addresses user\'s question without off-topic information',
+        'Completeness (0-100%): Includes all necessary information from documents',
+        'Citations/Attribution (0-100%): Proper citation of all claims',
+        'Length (0-100%): Appropriate detail without being too brief or verbose'
+    ]
+    for criterion in criteria_list:
+        doc.add_paragraph(criterion, style='List Bullet')
+    
+    doc.add_paragraph()  # Spacing
+    
+    # Add the actual validation prompt
+    doc.add_heading('Prompt Content:', 2)
+    validation_prompt = extract_validation_prompt()
+    
+    validation_para = doc.add_paragraph(validation_prompt)
+    validation_para.style = 'Normal'
+    
+    # Format the validation prompt text
+    for run in validation_para.runs:
+        run.font.size = Pt(10)
+        run.font.name = 'Courier New'
+    
+    # Save the document
+    output_path = 'HBV_AI_Assistant_System_Prompts.docx'
+    doc.save(output_path)
+    print(f"✓ Document created successfully: {output_path}")
+    print(f"✓ File location: {os.path.abspath(output_path)}")
+    print(f"\n📋 Exported Prompts:")
+    print(f"   1. HBV Assessment Prompt (core/hbv_assessment.py)")
+    print(f"   2. Agent System Prompt (core/agent.py)")
+    print(f"   3. Validation System Prompt (core/validation.py)")
+    
+    return output_path
+
+
+if __name__ == "__main__":
+    try:
+        create_prompts_document()
+    except Exception as e:
+        print(f"✗ Error creating document: {str(e)}")
+        import traceback
+        traceback.print_exc()

requirements.txt

@@ -0,0 +1,41 @@
+# API
+fastapi==0.116.1
+uvicorn==0.35.0
+
+# LangChain ecosystem - only what's needed
+langchain==0.3.27
+langchain_community==0.3.27
+langchain_openai==0.3.28
+langchain_huggingface==0.3.1
+langchain-pymupdf4llm==0.4.1
+
+# OpenAI
+openai==1.99.2
+
+# Data processing
+pandas==2.3.1
+pydantic==2.11.7
+
+# Utilities
+python-dotenv==1.1.1
+pytz==2025.2
+requests==2.32.4
+
+# Embeddings and search - CPU optimized versions
+sentence-transformers==5.0.0
+faiss-cpu==1.11.0.post1  # CPU version is much smaller
+rank-bm25==0.2.2
+
+# Fix for numpy and torch compatibility
+numpy<2
+torch==2.2.2+cpu
+--extra-index-url https://download.pytorch.org/whl/cpu
+
+# Document generation
+python-docx==1.1.2
+reportlab==4.2.5
+
+# Authentication
+python-multipart>=0.0.18
+itsdangerous==2.2.0
+

tempCodeRunnerFile.python

@@ -0,0 +1,146 @@
+# Import required libraries
+import pandas as pd
+from pathlib import Path
+from typing import List
+from langchain.schema import Document
+from core.config import logger
+from unstructured.partition.pdf import partition_pdf
+from unstructured.chunking.title import chunk_by_title
+
+
+def load_pdf_documents(pdf_path: Path) -> List[Document]:
+    """
+    Load and process PDF documents from medical guidelines using Unstructured.io.
+    Uses high-resolution strategy with ML-based table detection for borderless tables.
+    Extracts disease and provider from directory structure.
+   
+    Directory structure expected: data/new_data/PROVIDER/file.pdf
+    Example: data/new_data/SASLT/SASLT_2021.pdf
+   
+    Args:
+        pdf_path: Path to the PDF file
+       
+    Returns:
+        List of Document objects with metadata (source, disease, provider, page_number)
+    """
+    try:
+        # Validate file exists
+        if not pdf_path.exists():
+            raise FileNotFoundError(f"PDF file not found at {pdf_path}")
+       
+        # Extract provider from directory structure
+        # Structure: data/new_data/PROVIDER/file.pdf
+        path_parts = pdf_path.parts
+        disease = "HBV"  # Default disease for this system
+        provider = "unknown"
+       
+        # Find provider: it's the parent directory of the PDF file
+        if len(path_parts) >= 2:
+            provider = path_parts[-2]  # Parent directory (e.g., SASLT)
+           
+        # If provider is 'new_data', it means file is directly in new_data folder
+        if provider.lower() == "new_data":
+            provider = "unknown"
+       
+        # Use Unstructured.io to partition the PDF
+        # hi_res strategy uses ML models for better table detection
+        elements = partition_pdf(
+            filename=str(pdf_path),
+            strategy="hi_res",  # Use ML-based detection for borderless tables
+            infer_table_structure=True,  # Detect table structure without borders
+            extract_images_in_pdf=True,  # Extract images with OCR
+            languages=["eng"],  # OCR language
+            include_page_breaks=True,  # Maintain page boundaries
+        )
+       
+        # Group elements by page number
+        pages_content = {}
+        for element in elements:
+            # Get page number from metadata (1-indexed)
+            page_num = element.metadata.page_number if hasattr(element.metadata, 'page_number') else 1
+           
+            if page_num not in pages_content:
+                pages_content[page_num] = []
+           
+            # Convert element to text
+            pages_content[page_num].append(element.text)
+       
+        # Create Document objects for each page
+        documents = []
+        for page_num in sorted(pages_content.keys()):
+            # Combine all elements on the page
+            page_content = "\n\n".join(pages_content[page_num])
+           
+            if page_content.strip():
+                processed_doc = Document(
+                    page_content=page_content,
+                    metadata={
+                        "source": pdf_path.name,
+                        "disease": disease,
+                        "provider": provider,
+                        "page_number": page_num
+                    }
+                )
+                documents.append(processed_doc)
+       
+        logger.info(f"Loaded {len(documents)} pages from PDF: {pdf_path.name} (Disease: {disease}, Provider: {provider})")
+        return documents
+       
+    except Exception as e:
+        logger.error(f"Error loading PDF documents from {pdf_path}: {str(e)}")
+        raise
+
+
+# Alternative version: Preserve element types (useful for RAG)
+def load_pdf_documents_with_elements(pdf_path: Path) -> List[Document]:
+    """
+    Load PDF documents while preserving element types (text, table, title, etc.).
+    Useful for better RAG retrieval by maintaining document structure.
+    """
+    try:
+        if not pdf_path.exists():
+            raise FileNotFoundError(f"PDF file not found at {pdf_path}")
+       
+        path_parts = pdf_path.parts
+        disease = "HBV"
+        provider = path_parts[-2] if len(path_parts) >= 2 else "unknown"
+        if provider.lower() == "new_data":
+            provider = "unknown"
+       
+        elements = partition_pdf(
+            filename=str(pdf_path),
+            strategy="hi_res",
+            infer_table_structure=True,
+            extract_images_in_pdf=True,
+            languages=["eng"],
+        )
+       
+        documents = []
+        for idx, element in enumerate(elements):
+            if element.text.strip():
+                page_num = element.metadata.page_number if hasattr(element.metadata, 'page_number') else 1
+                element_type = element.category  # e.g., "Table", "Title", "NarrativeText"
+               
+                processed_doc = Document(
+                    page_content=element.text,
+                    metadata={
+                        "source": pdf_path.name,
+                        "disease": disease,
+                        "provider": provider,
+                        "page_number": page_num,
+                        "element_type": element_type,
+                        "element_id": idx
+                    }
+                )
+                documents.append(processed_doc)
+       
+        logger.info(f"Loaded {len(documents)} elements from PDF: {pdf_path.name} (Disease: {disease}, Provider: {provider})")
+        return documents
+       
+    except Exception as e:
+        logger.error(f"Error loading PDF documents from {pdf_path}: {str(e)}")
+        raise
+
+
+# Usage
+doc = load_pdf_documents(Path(r"data\processed_data\SASLT\SASLT 2021_20251026_171017.pdf"))