added readme

.dockerignore

@@ -0,0 +1,54 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# Environment variables
+.env
+.env.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Testing
+test/
+*.coverage
+htmlcov/
+.pytest_cache/
+
+# Git
+.git/
+.gitignore
+
+# UV lock file (runtime dependency)
+uv.lock
+
+# Python version file
+.python-version
+
+# Documentation (if not needed in container)
+API_DOCS.md
+REFACTORING_SUMMARY.md
+README.md
+
+# Logs
+*.log

.env

@@ -0,0 +1,11 @@
+# JWT Configuration
+# IMPORTANT: Generate a secure secret key for production
+# You can generate one using: openssl rand -hex 32
+JWT_SECRET_KEY=your-secret-key-change-in-production
+
+# MongoDB Configuration
+# Connection string for MongoDB database
+MONGODB_URI=mongodb+srv://arnavjagadeesh09_db_user:aP4x5QkUdSThzpxT@cluster0.uo44a8g.mongodb.net/?appName=Cluster0
+
+# MongoDB database name
+MONGODB_DB=afs

.env.example

@@ -0,0 +1,11 @@
+# JWT Configuration
+# IMPORTANT: Generate a secure secret key for production
+# You can generate one using: openssl rand -hex 32
+JWT_SECRET_KEY=your-secret-key-change-in-production
+
+# MongoDB Configuration
+# Connection string for MongoDB database
+MONGODB_URI=mongodb://localhost:27017
+
+# MongoDB database name
+MONGODB_DB=afs

.python-version

@@ -0,0 +1 @@
+3.13

API_DOCS.md

@@ -0,0 +1,243 @@
+# API Documentation
+
+## Overview
+This document describes the new API endpoints added to the AFS backend for face recognition and audio streaming.
+
+## Face Recognition APIs
+
+### 1. Upload 360-Degree Reference Video
+**Endpoint:** `POST /api/face/upload-video`
+
+**Description:** Upload a 360-degree reference video for face recognition training. The video will be processed to extract high-quality face embeddings.
+
+**Authentication:** Required (JWT token)
+
+**Request:**
+- Content-Type: `multipart/form-data`
+- Body: `file` (video file - .mp4, .avi, .mov, .mkv)
+
+**Response:**
+```json
+{
+  "ok": true,
+  "message": "Video processed successfully",
+  "frames_used": 10,
+  "embeddings_count": 1
+}
+```
+
+### 2. Upload Reference Image
+**Endpoint:** `POST /api/face/upload-image`
+
+**Description:** Upload a single reference image for face recognition.
+
+**Authentication:** Required (JWT token)
+
+**Request:**
+- Content-Type: `multipart/form-data`
+- Body: `file` (image file - .jpg, .jpeg, .png)
+
+**Response:**
+```json
+{
+  "ok": true,
+  "message": "Image processed successfully",
+  "embeddings_count": 1,
+  "saved_path": "/path/to/Model/ref_image.jpg"
+}
+```
+
+### 3. Get Cache Status
+**Endpoint:** `GET /api/face/cache-status`
+
+**Description:** Check if face recognition embeddings are cached and ready to use.
+
+**Authentication:** Required (JWT token)
+
+**Response (Cached):**
+```json
+{
+  "ok": true,
+  "cached": true,
+  "video_path": "my_scan.mp4",
+  "model_name": "ArcFace",
+  "num_frames_used": 10,
+  "version": 2
+}
+```
+
+**Response (Not Cached):**
+```json
+{
+  "ok": true,
+  "cached": false,
+  "message": "No cache found. Please upload a reference video or image."
+}
+```
+
+## Audio Streaming APIs
+
+### 1. Start Audio Stream
+**Endpoint:** `POST /api/audio/start-stream`
+
+**Description:** Start a new audio recording session. Returns a session ID for streaming.
+
+**Authentication:** Required (JWT token)
+
+**Request:**
+- Content-Type: `multipart/form-data`
+- Body:
+  - `sample_rate` (optional, default: 16000)
+  - `channels` (optional, default: 1 for mono, 2 for stereo)
+
+**Response:**
+```json
+{
+  "ok": true,
+  "session_id": "uuid-here",
+  "filename": "/path/to/Model/audio_recordings/audio_uuid_timestamp.wav",
+  "sample_rate": 16000,
+  "channels": 1
+}
+```
+
+### 2. Audio WebSocket Stream
+**Endpoint:** `WebSocket /ws/audio/{session_id}`
+
+**Description:** WebSocket endpoint for streaming audio data with optional angle information.
+
+**Authentication:** Not required at WebSocket level (use session_id from start-stream)
+
+**Send (Binary Audio Data):**
+```
+WebSocket Binary Message: raw audio bytes (16-bit PCM)
+```
+
+**Send (JSON with Angle):**
+```json
+{
+  "audio_data": "base64-encoded-audio-bytes",
+  "angle": 45.5
+}
+```
+
+**Send (Stop Command):**
+```json
+{
+  "command": "stop"
+}
+```
+
+**Receive:**
+```json
+{
+  "status": "received",
+  "bytes": 1024
+}
+```
+
+or
+
+```json
+{
+  "status": "received",
+  "angle": 45.5
+}
+```
+
+### 3. Stop Audio Stream
+**Endpoint:** `POST /api/audio/stop-stream/{session_id}`
+
+**Description:** Stop an active audio recording stream.
+
+**Authentication:** Required (JWT token)
+
+**Response:**
+```json
+{
+  "ok": true,
+  "message": "Audio stream stopped successfully"
+}
+```
+
+### 4. List Audio Recordings
+**Endpoint:** `GET /api/audio/recordings`
+
+**Description:** Get a list of all audio recordings.
+
+**Authentication:** Required (JWT token)
+
+**Response:**
+```json
+{
+  "ok": true,
+  "recordings": [
+    "/path/to/Model/audio_recordings/audio_uuid1_timestamp1.wav",
+    "/path/to/Model/audio_recordings/audio_uuid2_timestamp2.wav"
+  ],
+  "count": 2
+}
+```
+
+## File Storage
+
+All uploaded files and processed data are stored in the `/Model/` directory:
+
+- **Reference Videos:** `/Model/my_scan.mp4` (overwritten on each upload)
+- **Reference Images:** `/Model/ref_{filename}`
+- **Embeddings Cache:** `/Model/embeddings_cache.pkl`
+- **Audio Recordings:** `/Model/audio_recordings/audio_{session_id}_{timestamp}.wav`
+- **Audio Metadata:** `/Model/audio_recordings/audio_{session_id}_{timestamp}_metadata.txt`
+
+## Metadata Format
+
+Audio metadata files contain timestamp and angle data in CSV format:
+```
+timestamp,angle
+0.000,45.50
+0.064,46.20
+0.128,47.00
+```
+
+## Usage Example (Python)
+
+```python
+import requests
+import websockets
+import asyncio
+
+# 1. Upload reference video
+with open("my_360_scan.mp4", "rb") as f:
+    response = requests.post(
+        "http://localhost:8000/api/face/upload-video",
+        files={"file": f},
+        headers={"Authorization": f"Bearer {token}"}
+    )
+print(response.json())
+
+# 2. Start audio stream
+response = requests.post(
+    "http://localhost:8000/api/audio/start-stream",
+    data={"sample_rate": 16000, "channels": 1},
+    headers={"Authorization": f"Bearer {token}"}
+)
+session_id = response.json()["session_id"]
+
+# 3. Stream audio via WebSocket
+async def stream_audio():
+    uri = f"ws://localhost:8000/ws/audio/{session_id}"
+    async with websockets.connect(uri) as websocket:
+        # Send audio chunk with angle
+        await websocket.send(json.dumps({
+            "audio_data": base64.b64encode(audio_bytes).decode(),
+            "angle": 45.5
+        }))
+        
+        # Or send raw binary
+        await websocket.send(audio_bytes)
+        
+        # Stop when done
+        await websocket.send(json.dumps({"command": "stop"}))
+
+asyncio.run(stream_audio())
+```

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -0,0 +1,10 @@
+---
+title: AFS Backend
+emoji: 🚀
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: "4.31.0"
+app_file: app.py
+pinned: false
+---

REFACTORING_SUMMARY.md

@@ -0,0 +1,127 @@
+# Refactoring Summary
+
+## What Was Done
+
+### 1. **Model Directory Usage Analysis**
+The backend uses the following files from `/Model/` directory:
+- `embeddings_cache.pkl` - Face recognition embeddings cache
+- `yolov8n-face.pt` - YOLO face detection model
+- `my_scan.mp4` - Reference 360-degree scan video
+- `Adi.jpg` - Reference images
+
+Both `single_tracker.py` and `multi_tracker.py` access the Model directory.
+
+### 2. **Created New Services**
+
+#### `services/face_recognition.py`
+- Extracted face recognition logic from `Model/face_model.py`
+- Class: `FaceRecognitionService`
+- Methods:
+  - `extract_embeddings_from_video()` - Process 360° video with quality filtering
+  - `extract_embeddings_from_image()` - Process single reference image
+  - `save_embeddings_cache()` - Save processed embeddings
+  - `load_embeddings_cache()` - Load cached embeddings
+  - `calculate_blur_score()` - Image sharpness detection
+  - `calculate_frontal_score()` - Face frontality score
+
+#### `services/audio_processing.py`
+- New service for audio streaming with angle data
+- Class: `AudioProcessor`
+- Methods:
+  - `create_audio_stream()` - Start new recording session
+  - `write_audio_chunk()` - Write audio with optional angle metadata
+  - `close_audio_stream()` - Finalize recording
+  - `get_audio_files()` - List all recordings
+
+### 3. **Added API Endpoints to `server.py`**
+
+#### Face Recognition APIs:
+- `POST /api/face/upload-video` - Upload 360° reference video
+- `POST /api/face/upload-image` - Upload reference image
+- `GET /api/face/cache-status` - Check embeddings cache status
+
+#### Audio Streaming APIs:
+- `POST /api/audio/start-stream` - Start audio recording session
+- `WebSocket /ws/audio/{session_id}` - Stream audio with angle data
+- `POST /api/audio/stop-stream/{session_id}` - Stop recording
+- `GET /api/audio/recordings` - List all recordings
+
+### 4. **File Storage Structure**
+```
+/Model/
+├── my_scan.mp4                    # Reference video (uploaded via API)
+├── ref_*.jpg                      # Reference images (uploaded via API)
+├── embeddings_cache.pkl           # Processed face embeddings
+├── yolov8n-face.pt               # YOLO model (static)
+└── audio_recordings/
+    ├── audio_{uuid}_{timestamp}.wav           # Audio recording
+    └── audio_{uuid}_{timestamp}_metadata.txt  # Angle metadata (CSV)
+```
+
+### 5. **Audio Metadata Format**
+The metadata file stores timestamp and angle in CSV format:
+```csv
+timestamp,angle
+0.000,45.50
+0.064,46.20
+0.128,47.00
+```
+
+## How to Use
+
+### Upload 360-Degree Video:
+```bash
+curl -X POST "http://localhost:8000/api/face/upload-video" \
+  -H "Authorization: Bearer YOUR_TOKEN" \
+  -F "file=@my_360_scan.mp4"
+```
+
+### Upload Reference Image:
+```bash
+curl -X POST "http://localhost:8000/api/face/upload-image" \
+  -H "Authorization: Bearer YOUR_TOKEN" \
+  -F "file=@reference.jpg"
+```
+
+### Start Audio Stream:
+```bash
+# 1. Start stream (get session_id)
+curl -X POST "http://localhost:8000/api/audio/start-stream" \
+  -H "Authorization: Bearer YOUR_TOKEN" \
+  -F "sample_rate=16000" \
+  -F "channels=1"
+
+# 2. Connect via WebSocket and stream
+# ws://localhost:8000/ws/audio/{session_id}
+
+# 3. Send audio chunks (binary or JSON with angle)
+# Binary: raw 16-bit PCM audio bytes
+# JSON: {"audio_data": "base64...", "angle": 45.5}
+
+# 4. Stop: {"command": "stop"}
+```
+
+## Key Features
+
+1. **Quality Filtering**: Video processing uses blur detection and frontal face scoring to select best frames
+2. **Temporal Spacing**: Selects frames evenly distributed across the video for comprehensive coverage
+3. **Angle Tracking**: Audio streams can include direction/angle metadata for spatial audio analysis
+4. **Mono/Stereo Support**: Configurable audio channels (1 or 2)
+5. **Authentication**: All endpoints protected with JWT tokens
+6. **Async Processing**: CPU-intensive tasks run in thread pool executor
+
+## Original face_model.py
+
+The original file at `/Model/face_model.py` remains unchanged and can still be run standalone for testing or manual processing. The new API provides the same functionality but in a service-oriented architecture accessible via HTTP/WebSocket.
+
+## Dependencies
+
+All required packages are already in `requirements.txt`:
+- FastAPI, Uvicorn
+- OpenCV (cv2)
+- DeepFace
+- Ultralytics (YOLO)
+- NumPy
+- Wave (stdlib)
+
+No additional dependencies needed!

pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "backend"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "deepface>=0.0.99",
+    "fastapi>=0.135.3",
+    "pymongo>=4.9.0",
+    "numpy>=2.4.4",
+    "opencv-python>=4.13.0.92",
+    "passlib[bcrypt]>=1.7.4",
+    "python-jose[cryptography]>=3.5.0",
+    "tf-keras>=2.21.0",
+    "ultralytics>=8.4.33",
+    "uvicorn>=0.43.0",
+    "websockets>=16.0",
+    "lap>=0.5.13",
+    "bcrypt>=5.0.0",
+    "python-dotenv>=1.2.2",
+    "python-multipart>=0.0.22",
+]

requirements.txt

@@ -5,3 +5,8 @@ opencv-python
 numpy
 ultralytics
 deepface
+pymongo>=4.9.0
+passlib[bcrypt]
+bcrypt
+python-jose[cryptography]
+python-dotenv

server.py

@@ -1,19 +1,35 @@
-from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, status, Depends, UploadFile, File, Form
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 import uvicorn
 import cv2
 import numpy as np
-import base64
 import json
 import logging
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime
+from datetime import datetime, timedelta
 import threading
+import os
+import base64
+import hashlib
+from pydantic import BaseModel, Field
+from pymongo import AsyncMongoClient
+from passlib.context import CryptContext
+from jose import JWTError, jwt
+from dotenv import load_dotenv
+from pathlib import Path
+import shutil
+import uuid
 
 from services.single_tracker import SingleTracker
 from services.multi_tracker import MultiTracker
+from services.face_recognition import FaceRecognitionService
+from services.audio_processing import AudioProcessor
+
+# Load environment variables from .env file
+load_dotenv()
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
@@ -49,9 +65,125 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# Initialize trackers
+# Initialize trackers and services
+MODEL_DIR = Path(__file__).parent.parent / "Model"
 single_tracker = SingleTracker()
 multi_tracker = MultiTracker()
+face_service = FaceRecognitionService(str(MODEL_DIR))
+audio_processor = AudioProcessor(str(MODEL_DIR))
+
+# MongoDB state
+mongo_client: AsyncMongoClient | None = None
+users_collection = None
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+# JWT Configuration
+SECRET_KEY = os.getenv("JWT_SECRET_KEY", "your-secret-key-change-in-production")
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 24 * 7  # 7 days
+
+security = HTTPBearer()
+
+
+class RegisterRequest(BaseModel):
+    full_name: str = Field(min_length=2, max_length=80)
+    email: str = Field(min_length=5, max_length=254)
+    password: str = Field(min_length=8, max_length=128)
+
+
+class LoginRequest(BaseModel):
+    email: str = Field(min_length=5, max_length=254)
+    password: str = Field(min_length=8, max_length=128)
+
+
+class UserPublic(BaseModel):
+    id: str
+    full_name: str
+    email: str
+
+
+class AuthResponse(BaseModel):
+    ok: bool
+    message: str
+    user: UserPublic
+    token: str
+
+
+def normalize_email(email: str) -> str:
+    return email.strip().lower()
+
+
+def get_password_hash(password: str) -> str:
+    # Hash password with SHA256 first to handle any length, then use bcrypt
+    password_hash = hashlib.sha256(password.encode('utf-8')).hexdigest()
+    return pwd_context.hash(password_hash)
+
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    # Apply same SHA256 transformation before verifying
+    password_hash = hashlib.sha256(plain_password.encode('utf-8')).hexdigest()
+    return pwd_context.verify(password_hash, hashed_password)
+
+
+def require_users_collection():
+    if users_collection is None:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Database is not initialized yet. Please retry.",
+        )
+    return users_collection
+
+
+def create_access_token(data: dict, expires_delta: timedelta | None = None):
+    to_encode = data.copy()
+    if expires_delta:
+        expire = datetime.utcnow() + expires_delta
+    else:
+        expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
+    to_encode.update({"exp": expire})
+    encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
+    return encoded_jwt
+
+
+async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    collection = require_users_collection()
+    token = credentials.credentials
+    
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        user_id: str = payload.get("sub")
+        if user_id is None:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Invalid authentication credentials",
+            )
+    except JWTError:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired token",
+        )
+    
+    from bson import ObjectId
+    try:
+        user_doc = await collection.find_one({"_id": ObjectId(user_id)})
+    except:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="User not found",
+        )
+    
+    if user_doc is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="User not found",
+        )
+    
+    return UserPublic(
+        id=str(user_doc["_id"]),
+        full_name=user_doc["full_name"],
+        email=user_doc["email"],
+    )
 
 def decode_binary_image(img_data: bytes):
     """Decodes raw JPEG bytes into an OpenCV numpy array."""
@@ -186,8 +318,95 @@ async def vcam_generator_loop():
 
 @app.on_event("startup")
 async def startup_event():
+    global mongo_client, users_collection
+    mongo_uri = os.getenv("MONGODB_URI", "mongodb://localhost:27017")
+    mongo_db_name = os.getenv("MONGODB_DB", "afs")
+
+    mongo_client = AsyncMongoClient(mongo_uri)
+    users_collection = mongo_client[mongo_db_name]["users"]
+    await users_collection.create_index("email", unique=True)
+    logger.info("Connected to MongoDB and initialized users index.")
+
     asyncio.create_task(vcam_generator_loop())
 
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    global mongo_client
+    if mongo_client is not None:
+        mongo_client.close()
+        logger.info("MongoDB connection closed.")
+
+
+@app.post("/auth/register", response_model=AuthResponse)
+async def register(payload: RegisterRequest):
+    collection = require_users_collection()
+    email = normalize_email(payload.email)
+
+    existing_user = await collection.find_one({"email": email})
+    if existing_user:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail="An account with this email already exists.",
+        )
+
+    now = datetime.utcnow()
+    user_doc = {
+        "full_name": payload.full_name.strip(),
+        "email": email,
+        "password_hash": get_password_hash(payload.password),
+        "created_at": now,
+        "updated_at": now,
+    }
+    insert_result = await collection.insert_one(user_doc)
+    
+    user_id = str(insert_result.inserted_id)
+    access_token = create_access_token(data={"sub": user_id})
+
+    return AuthResponse(
+        ok=True,
+        message="Account created successfully.",
+        user=UserPublic(
+            id=user_id,
+            full_name=user_doc["full_name"],
+            email=user_doc["email"],
+        ),
+        token=access_token,
+    )
+
+
+@app.post("/auth/login", response_model=AuthResponse)
+async def login(payload: LoginRequest):
+    collection = require_users_collection()
+    email = normalize_email(payload.email)
+
+    user_doc = await collection.find_one({"email": email})
+    if not user_doc or not verify_password(payload.password, user_doc["password_hash"]):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid email or password.",
+        )
+    
+    user_id = str(user_doc["_id"])
+    access_token = create_access_token(data={"sub": user_id})
+
+    return AuthResponse(
+        ok=True,
+        message="Login successful.",
+        user=UserPublic(
+            id=user_id,
+            full_name=user_doc["full_name"],
+            email=user_doc["email"],
+        ),
+        token=access_token,
+    )
+
+
+@app.get("/auth/verify", response_model=UserPublic)
+async def verify_token(current_user: UserPublic = Depends(get_current_user)):
+    """Verify JWT token and return user info"""
+    return current_user
+
 @app.websocket("/ws")
 async def websocket_endpoint(websocket: WebSocket):
     global is_recording, video_writer, recording_filename, latest_obs_frame, is_obs_active
@@ -312,5 +531,184 @@ async def websocket_endpoint(websocket: WebSocket):
             video_writer = None
         is_recording = False
 
+# === FACE RECOGNITION ENDPOINTS ===
+
+@app.post("/api/face/upload-video")
+async def upload_reference_video(
+    file: UploadFile = File(...),
+    current_user: UserPublic = Depends(get_current_user)
+):
+    """Upload a 360-degree reference video for face recognition training."""
+    if not file.filename.endswith(('.mp4', '.avi', '.mov', '.mkv')):
+        raise HTTPException(status_code=400, detail="Invalid video format. Use mp4, avi, mov, or mkv")
+    
+    video_path = MODEL_DIR / "my_scan.mp4"
+    
+    try:
+        with open(video_path, 'wb') as f:
+            shutil.copyfileobj(file.file, f)
+        
+        embeddings, num_frames = await asyncio.get_event_loop().run_in_executor(
+            executor, face_service.extract_embeddings_from_video, str(video_path)
+        )
+        
+        face_service.save_embeddings_cache(embeddings, str(video_path), num_frames)
+        
+        return {
+            "ok": True,
+            "message": "Video processed successfully",
+            "frames_used": num_frames,
+            "embeddings_count": len(embeddings)
+        }
+    except Exception as e:
+        logger.error(f"Error processing video: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/face/upload-image")
+async def upload_reference_image(
+    file: UploadFile = File(...),
+    current_user: UserPublic = Depends(get_current_user)
+):
+    """Upload a reference image for face recognition."""
+    if not file.filename.endswith(('.jpg', '.jpeg', '.png')):
+        raise HTTPException(status_code=400, detail="Invalid image format. Use jpg, jpeg, or png")
+    
+    image_path = MODEL_DIR / f"ref_{file.filename}"
+    
+    try:
+        with open(image_path, 'wb') as f:
+            shutil.copyfileobj(file.file, f)
+        
+        embeddings = await asyncio.get_event_loop().run_in_executor(
+            executor, face_service.extract_embeddings_from_image, str(image_path)
+        )
+        
+        return {
+            "ok": True,
+            "message": "Image processed successfully",
+            "embeddings_count": len(embeddings),
+            "saved_path": str(image_path)
+        }
+    except Exception as e:
+        logger.error(f"Error processing image: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/face/cache-status")
+async def get_cache_status(current_user: UserPublic = Depends(get_current_user)):
+    """Get the current face recognition cache status."""
+    cache_data = face_service.load_embeddings_cache()
+    
+    if cache_data:
+        return {
+            "ok": True,
+            "cached": True,
+            "video_path": cache_data.get('video_path'),
+            "model_name": cache_data.get('model_name'),
+            "num_frames_used": cache_data.get('num_frames_used'),
+            "version": cache_data.get('version')
+        }
+    else:
+        return {
+            "ok": True,
+            "cached": False,
+            "message": "No cache found. Please upload a reference video or image."
+        }
+
+# === AUDIO STREAMING ENDPOINTS ===
+
+@app.post("/api/audio/start-stream")
+async def start_audio_stream(
+    sample_rate: int = Form(16000),
+    channels: int = Form(1),
+    current_user: UserPublic = Depends(get_current_user)
+):
+    """Start a new audio recording stream."""
+    session_id = str(uuid.uuid4())
+    
+    try:
+        filename = audio_processor.create_audio_stream(session_id, sample_rate, channels)
+        return {
+            "ok": True,
+            "session_id": session_id,
+            "filename": filename,
+            "sample_rate": sample_rate,
+            "channels": channels
+        }
+    except Exception as e:
+        logger.error(f"Error starting audio stream: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.websocket("/ws/audio/{session_id}")
+async def websocket_audio_stream(websocket: WebSocket, session_id: str):
+    """WebSocket endpoint for streaming audio with angle data."""
+    await websocket.accept()
+    logger.info(f"Audio WebSocket connection established for session {session_id}")
+    
+    try:
+        while True:
+            message = await websocket.receive()
+            
+            if "bytes" in message:
+                audio_data = message["bytes"]
+                audio_processor.write_audio_chunk(session_id, audio_data)
+                await websocket.send_json({"status": "received", "bytes": len(audio_data)})
+            
+            elif "text" in message:
+                try:
+                    payload = json.loads(message["text"])
+                    
+                    if "audio_data" in payload and "angle" in payload:
+                        audio_bytes = base64.b64decode(payload["audio_data"])
+                        angle = float(payload["angle"])
+                        audio_processor.write_audio_chunk(session_id, audio_bytes, angle)
+                        await websocket.send_json({"status": "received", "angle": angle})
+                    
+                    elif payload.get("command") == "stop":
+                        audio_processor.close_audio_stream(session_id)
+                        await websocket.send_json({"status": "stopped", "message": "Stream closed"})
+                        break
+                        
+                except json.JSONDecodeError:
+                    logger.error("Invalid JSON in audio stream")
+                    
+    except WebSocketDisconnect:
+        logger.info(f"Audio WebSocket client disconnected for session {session_id}")
+        if session_id in audio_processor.active_streams:
+            audio_processor.close_audio_stream(session_id)
+    except Exception as e:
+        logger.error(f"Audio WebSocket error: {e}")
+        if session_id in audio_processor.active_streams:
+            audio_processor.close_audio_stream(session_id)
+
+@app.post("/api/audio/stop-stream/{session_id}")
+async def stop_audio_stream(
+    session_id: str,
+    current_user: UserPublic = Depends(get_current_user)
+):
+    """Stop an active audio recording stream."""
+    try:
+        audio_processor.close_audio_stream(session_id)
+        return {
+            "ok": True,
+            "message": "Audio stream stopped successfully"
+        }
+    except Exception as e:
+        logger.error(f"Error stopping audio stream: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/audio/recordings")
+async def list_audio_recordings(current_user: UserPublic = Depends(get_current_user)):
+    """List all audio recordings."""
+    try:
+        recordings = audio_processor.get_audio_files()
+        return {
+            "ok": True,
+            "recordings": recordings,
+            "count": len(recordings)
+        }
+    except Exception as e:
+        logger.error(f"Error listing recordings: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
 if __name__ == "__main__":
     uvicorn.run("server:app", host="0.0.0.0", port=8000, reload=True)

services/audio_processing.py

@@ -0,0 +1,69 @@
+import wave
+import numpy as np
+import logging
+from pathlib import Path
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+class AudioProcessor:
+    def __init__(self, model_dir: str):
+        self.model_dir = Path(model_dir)
+        self.audio_dir = self.model_dir / "audio_recordings"
+        self.audio_dir.mkdir(exist_ok=True)
+        
+        self.active_streams = {}
+    
+    def create_audio_stream(self, session_id: str, sample_rate: int = 16000, channels: int = 1):
+        """Create a new audio recording stream."""
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = self.audio_dir / f"audio_{session_id}_{timestamp}.wav"
+        metadata_file = self.audio_dir / f"audio_{session_id}_{timestamp}_metadata.txt"
+        
+        wav_file = wave.open(str(filename), 'wb')
+        wav_file.setnchannels(channels)
+        wav_file.setsampwidth(2)
+        wav_file.setframerate(sample_rate)
+        
+        self.active_streams[session_id] = {
+            'wav_file': wav_file,
+            'metadata_file': metadata_file,
+            'metadata_handle': open(metadata_file, 'w'),
+            'sample_rate': sample_rate,
+            'channels': channels,
+            'frame_count': 0
+        }
+        
+        logger.info(f"Created audio stream {session_id} -> {filename}")
+        return str(filename)
+    
+    def write_audio_chunk(self, session_id: str, audio_data: bytes, angle: float = None):
+        """Write audio chunk with optional angle metadata."""
+        if session_id not in self.active_streams:
+            raise ValueError(f"No active stream for session {session_id}")
+        
+        stream = self.active_streams[session_id]
+        stream['wav_file'].writeframes(audio_data)
+        
+        if angle is not None:
+            timestamp = stream['frame_count'] / stream['sample_rate']
+            stream['metadata_handle'].write(f"{timestamp:.3f},{angle:.2f}\n")
+        
+        stream['frame_count'] += len(audio_data) // (2 * stream['channels'])
+    
+    def close_audio_stream(self, session_id: str):
+        """Close and finalize audio stream."""
+        if session_id not in self.active_streams:
+            raise ValueError(f"No active stream for session {session_id}")
+        
+        stream = self.active_streams[session_id]
+        stream['wav_file'].close()
+        stream['metadata_handle'].close()
+        
+        logger.info(f"Closed audio stream {session_id}")
+        del self.active_streams[session_id]
+    
+    def get_audio_files(self):
+        """List all audio recordings."""
+        wav_files = list(self.audio_dir.glob("*.wav"))
+        return [str(f) for f in sorted(wav_files, reverse=True)]

services/face_recognition.py

@@ -0,0 +1,160 @@
+import cv2
+from ultralytics import YOLO
+from deepface import DeepFace
+import numpy as np
+import pickle
+import os
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+class FaceRecognitionService:
+    def __init__(self, model_dir: str):
+        self.model_dir = Path(model_dir)
+        self.model_name = "ArcFace"
+        self.detector_model = "yolov8n-face.pt"
+        self.cache_file = self.model_dir / "embeddings_cache.pkl"
+        
+        self.num_best_frames = 10
+        self.min_blur_threshold = 10
+        self.blur_weight = 0.6
+        self.frontal_weight = 0.4
+    
+    def calculate_blur_score(self, image):
+        """Calculate sharpness using Laplacian variance."""
+        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        laplacian_var = cv2.Laplacian(gray, cv2.CV_64F).var()
+        return laplacian_var
+    
+    def calculate_frontal_score(self, face_data):
+        """Calculate how frontal the face is based on facial area."""
+        try:
+            facial_area = face_data.get('facial_area', {})
+            area = facial_area.get('w', 0) * facial_area.get('h', 0)
+            frontal_score = min(area / 50000.0, 1.0) * 100
+            return frontal_score
+        except:
+            return 50.0
+    
+    def extract_embeddings_from_video(self, video_path: str):
+        """Extract high-quality face embeddings from a 360-degree scan video."""
+        logger.info(f"Processing reference video scan: {video_path}")
+        logger.info("Phase 1: Analyzing frame quality...")
+        
+        cap_ref = cv2.VideoCapture(video_path)
+        total_frames = int(cap_ref.get(cv2.CAP_PROP_FRAME_COUNT))
+        
+        candidate_frames = []
+        frame_idx = 0
+        
+        while cap_ref.isOpened():
+            ret, frame = cap_ref.read()
+            if not ret:
+                break
+            
+            if frame_idx % 15 == 0:
+                blur_score = self.calculate_blur_score(frame)
+                
+                if blur_score <= self.min_blur_threshold:
+                    logger.debug(f"Frame {frame_idx}: Blur={blur_score:.1f} (too blurry, skipped)")
+                
+                if blur_score > self.min_blur_threshold:
+                    try:
+                        face_data = DeepFace.represent(frame, model_name=self.model_name, enforce_detection=True)[0]
+                        frontal_score = self.calculate_frontal_score(face_data)
+                        quality_score = (blur_score * self.blur_weight) + (frontal_score * self.frontal_weight)
+                        
+                        candidate_frames.append({
+                            'frame_idx': frame_idx,
+                            'frame': frame.copy(),
+                            'blur_score': blur_score,
+                            'frontal_score': frontal_score,
+                            'quality_score': quality_score,
+                            'embedding': face_data['embedding']
+                        })
+                        
+                        logger.debug(f"Frame {frame_idx}: Quality={quality_score:.1f}")
+                    except Exception as e:
+                        logger.debug(f"Frame {frame_idx}: No face detected - {e}")
+            
+            frame_idx += 1
+        
+        cap_ref.release()
+        
+        if not candidate_frames:
+            raise ValueError("No valid frames found in video")
+        
+        logger.info(f"Phase 2: Selecting top {self.num_best_frames} frames with temporal spacing...")
+        candidate_frames.sort(key=lambda x: x['quality_score'], reverse=True)
+        
+        segment_size = total_frames // self.num_best_frames
+        selected_frames = []
+        
+        for segment_idx in range(self.num_best_frames):
+            segment_start = segment_idx * segment_size
+            segment_end = (segment_idx + 1) * segment_size
+            
+            best_in_segment = None
+            best_quality = -1
+            
+            for candidate in candidate_frames:
+                if segment_start <= candidate['frame_idx'] < segment_end:
+                    if candidate['quality_score'] > best_quality:
+                        best_quality = candidate['quality_score']
+                        best_in_segment = candidate
+            
+            if best_in_segment:
+                selected_frames.append(best_in_segment)
+                logger.debug(f"Segment {segment_idx+1}: Frame {best_in_segment['frame_idx']}")
+        
+        if len(selected_frames) < self.num_best_frames:
+            for candidate in candidate_frames:
+                if candidate not in selected_frames:
+                    selected_frames.append(candidate)
+                    if len(selected_frames) >= self.num_best_frames:
+                        break
+        
+        logger.info(f"Phase 3: Averaging {len(selected_frames)} embeddings...")
+        embeddings_to_average = [frame['embedding'] for frame in selected_frames]
+        master_embedding = np.mean(embeddings_to_average, axis=0).tolist()
+        
+        return [master_embedding], len(selected_frames)
+    
+    def extract_embeddings_from_image(self, image_path: str):
+        """Extract face embedding from a single image."""
+        try:
+            embedding = DeepFace.represent(img_path=image_path, model_name=self.model_name)[0]["embedding"]
+            return [embedding]
+        except Exception as e:
+            logger.error(f"Could not extract embedding from {image_path}: {e}")
+            raise
+    
+    def save_embeddings_cache(self, embeddings, video_path: str, num_frames_used: int):
+        """Save embeddings to cache file."""
+        cache_data = {
+            'video_path': video_path,
+            'video_mtime': os.path.getmtime(video_path) if os.path.exists(video_path) else None,
+            'model_name': self.model_name,
+            'embeddings': embeddings,
+            'version': 2,
+            'num_frames_used': num_frames_used
+        }
+        
+        with open(self.cache_file, 'wb') as f:
+            pickle.dump(cache_data, f)
+        
+        logger.info(f"Saved embeddings cache to {self.cache_file}")
+    
+    def load_embeddings_cache(self):
+        """Load embeddings from cache file."""
+        if not os.path.exists(self.cache_file):
+            return None
+        
+        try:
+            with open(self.cache_file, 'rb') as f:
+                cache_data = pickle.load(f)
+            return cache_data
+        except Exception as e:
+            logger.error(f"Could not load cache: {e}")
+            return None

test/test_mongodb.py

@@ -0,0 +1,38 @@
+"""Test MongoDB connection to verify configuration"""
+import os
+import asyncio
+from dotenv import load_dotenv
+from pymongo import AsyncMongoClient
+
+async def test_mongodb_connection():
+    # Load environment variables
+    load_dotenv()
+    
+    mongo_uri = os.getenv("MONGODB_URI", "mongodb://localhost:27017")
+    mongo_db_name = os.getenv("MONGODB_DB", "afs")
+    
+    print(f"Testing MongoDB connection...")
+    print(f"URI: {mongo_uri[:20]}... (truncated for security)")
+    print(f"Database: {mongo_db_name}")
+    
+    try:
+        client = AsyncMongoClient(mongo_uri)
+        # Test the connection
+        await client.admin.command('ping')
+        print("✓ Successfully connected to MongoDB!")
+        
+        # Test database access
+        db = client[mongo_db_name]
+        collections = await db.list_collection_names()
+        print(f"✓ Database '{mongo_db_name}' accessible")
+        print(f"  Collections: {collections if collections else '(none)'}")
+        
+        client.close()
+        print("✓ Connection closed successfully")
+        return True
+    except Exception as e:
+        print(f"✗ Connection failed: {e}")
+        return False
+
+if __name__ == "__main__":
+    asyncio.run(test_mongodb_connection())

test/test_ndi.py

@@ -0,0 +1,21 @@
+from ctypes import cdll, POINTER, c_int, c_void_p, Structure, c_char_p, byref
+import sys
+
+print("Loading NDI dylib directly...")
+try:
+    libndi = cdll.LoadLibrary("/usr/local/lib/libndi.dylib")
+    print("Found NDI in /usr/local/lib")
+except OSError:
+    try:
+        libndi = cdll.LoadLibrary("/Library/NDI SDK for Apple/lib/macOS/libndi.dylib")
+        print("Found NDI in SDK path")
+    except OSError as e:
+        print("NDI NOT FOUND:", e)
+        sys.exit(1)
+
+# Make sure basic initialization works
+libndi.NDIlib_initialize.restype = c_int
+result = libndi.NDIlib_initialize()
+print(f"NDIlib_initialize returned: {result}")
+if result:
+    print("NDI initialized perfectly via ctypes bindings!")

test/test_vcam.py

@@ -0,0 +1,9 @@
+import pyvirtualcam
+import numpy as np
+
+try:
+    with pyvirtualcam.Camera(width=1280, height=720, fps=30) as cam:
+        print(f"Virtual camera started: {cam.device} ({cam.width}x{cam.height} @ {cam.fps}fps)")
+        cam.send(np.zeros((720, 1280, 4), np.uint8))
+except Exception as e:
+    print(f"FAILED VCAM: {e}")

test/test_writer.py

@@ -0,0 +1,12 @@
+import cv2
+import numpy as np
+try:
+    fourcc = cv2.VideoWriter_fourcc(*'avc1')
+    out = cv2.VideoWriter('test_avc1.mp4', fourcc, 5.0, (640, 480))
+    for i in range(10):
+        frame = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8)
+        out.write(frame)
+    out.release()
+    print("avc1 SUCCESS")
+except Exception as e:
+    print(f"FAILED: {e}")