Initial clean commit - FastAPI movie backend without large data files

.env.example

@@ -0,0 +1,14 @@
+# OpenAI API key for embeddings
+OPENAI_API_KEY=your_openai_api_key_here
+
+# TMDB API key for movie data
+TMDB_API_KEY=your_tmdb_api_key_here
+
+# API authentication token
+API_TOKEN=your_api_token_here
+
+# Environment (dev/prod)
+ENV=dev
+
+# Logging level
+LOG_LEVEL=INFO

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.index filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,57 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Environment
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Data files (these are large and will be generated on deployment)
+app/data/*.npy
+app/data/*.index
+app/data/movie_metadata.json
+app/data/id_map.json
+app/data/checkpoints/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Temporary files
+tmp/
+temp/

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.9-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements first for better caching
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY app/ ./app/
+
+# Create data directory
+RUN mkdir -p app/data
+
+# Expose port
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:7860/health || exit 1
+
+# Run the application
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,42 @@
+---
+title: Karl Movie Vector Backend
+emoji: 🎬
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+license: mit
+---
+
+# Karl Movie Vector Backend
+
+FastAPI backend for semantic movie recommendations using FAISS and OpenAI embeddings. Powers intelligent movie discovery with geometric subspace algorithms.
+
+## Features
+
+- Semantic movie search using OpenAI embeddings
+- FAISS-powered vector similarity search
+- Geometric subspace algorithms for multi-movie preferences
+- ~150ms response time on CPU
+- RESTful API with Bearer token authentication
+
+## API Usage
+
+```bash
+curl -X POST "https://yonnel-karl-movie-vector-backend.hf.space/explore" \
+  -H "Authorization: Bearer YOUR_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "liked_ids": [550, 680],
+    "disliked_ids": [],
+    "top_k": 100
+  }'
+```
+
+## Environment Variables
+
+Set these in your Space settings:
+- `OPENAI_API_KEY`: Your OpenAI API key
+- `TMDB_API_KEY`: Your TMDB API key  
+- `API_TOKEN`: Authentication token for API access
+- `ENV`: Set to "prod" for production

README_HF.md

@@ -0,0 +1,42 @@
+---
+title: Karl Movie Vector Backend
+emoji: 🎬
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+license: mit
+---
+
+# Karl Movie Vector Backend
+
+FastAPI backend for semantic movie recommendations using FAISS and OpenAI embeddings. Powers intelligent movie discovery with geometric subspace algorithms.
+
+## Features
+
+- Semantic movie search using OpenAI embeddings
+- FAISS-powered vector similarity search
+- Geometric subspace algorithms for multi-movie preferences
+- ~150ms response time on CPU
+- RESTful API with Bearer token authentication
+
+## API Usage
+
+```bash
+curl -X POST "https://yonnel-karl-movie-vector-backend.hf.space/explore" \
+  -H "Authorization: Bearer YOUR_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "liked_ids": [550, 680],
+    "disliked_ids": [],
+    "top_k": 100
+  }'
+```
+
+## Environment Variables
+
+Set these in your Space settings:
+- `OPENAI_API_KEY`: Your OpenAI API key
+- `TMDB_API_KEY`: Your TMDB API key  
+- `API_TOKEN`: Authentication token for API access
+- `ENV`: Set to "prod" for production

app/__init__.py

@@ -0,0 +1,6 @@
+"""
+Karl-Movie Vector Backend
+A FastAPI service for semantic movie recommendations using FAISS and OpenAI embeddings
+"""
+
+__version__ = "1.0.0"

app/build_index.py

@@ -0,0 +1,485 @@
+"""
+Build FAISS index from movie embeddings
+This script should be run once to create the data files needed by the API
+"""
+import os
+import json
+import numpy as np
+import faiss
+from openai import OpenAI
+import requests
+from typing import Dict, List, Optional
+import time
+import argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import logging
+from settings import get_settings
+import pickle
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Checkpoint file paths
+CHECKPOINT_DIR = "app/data/checkpoints"
+MOVIE_DATA_CHECKPOINT = f"{CHECKPOINT_DIR}/movie_data.pkl"
+EMBEDDINGS_CHECKPOINT = f"{CHECKPOINT_DIR}/embeddings_progress.pkl"
+METADATA_CHECKPOINT = f"{CHECKPOINT_DIR}/metadata_progress.pkl"
+
+def save_checkpoint(data, filepath: str):
+    """Save checkpoint data to file"""
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+    with open(filepath, 'wb') as f:
+        pickle.dump(data, f)
+    logger.info(f"Checkpoint saved: {filepath}")
+
+def load_checkpoint(filepath: str):
+    """Load checkpoint data from file"""
+    if os.path.exists(filepath):
+        with open(filepath, 'rb') as f:
+            data = pickle.load(f)
+        logger.info(f"Checkpoint loaded: {filepath}")
+        return data
+    return None
+
+def cleanup_checkpoints():
+    """Remove checkpoint files after successful completion"""
+    import shutil
+    if os.path.exists(CHECKPOINT_DIR):
+        shutil.rmtree(CHECKPOINT_DIR)
+        logger.info("Checkpoint files cleaned up")
+
+class TMDBClient:
+    """Client for TMDB API with retry and backoff"""
+    
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.base_url = "https://api.themoviedb.org/3"
+        self.session = requests.Session()
+        
+    def _make_request(self, endpoint: str, params: dict = None, max_retries: int = 3) -> Optional[dict]:
+        """Make API request with retry and backoff"""
+        if params is None:
+            params = {}
+        params['api_key'] = self.api_key
+        
+        url = f"{self.base_url}{endpoint}"
+        
+        for attempt in range(max_retries):
+            try:
+                response = self.session.get(url, params=params, timeout=10)
+                
+                if response.status_code == 200:
+                    return response.json()
+                elif response.status_code == 429:
+                    # Rate limit - wait and retry
+                    wait_time = 2 ** attempt
+                    logger.warning(f"Rate limited, waiting {wait_time}s before retry...")
+                    time.sleep(wait_time)
+                    continue
+                elif response.status_code == 404:
+                    logger.warning(f"Resource not found: {url}")
+                    return None
+                else:
+                    logger.error(f"API error {response.status_code}: {response.text}")
+                    
+            except requests.exceptions.RequestException as e:
+                logger.error(f"Request failed (attempt {attempt + 1}): {e}")
+                if attempt < max_retries - 1:
+                    time.sleep(2 ** attempt)
+                    
+        return None
+    
+    def get_popular_movies(self, max_pages: int = 100) -> List[int]:
+        """Get movie IDs from popular movies pagination"""
+        movie_ids = []
+        
+        for page in range(1, max_pages + 1):
+            logger.info(f"Fetching popular movies page {page}/{max_pages}")
+            
+            data = self._make_request("/movie/popular", {"page": page})
+            if not data:
+                logger.error(f"Failed to fetch page {page}")
+                break
+                
+            # Check if we've exceeded total pages
+            if page > data.get('total_pages', 0):
+                logger.info(f"Reached last page ({data.get('total_pages')})")
+                break
+                
+            # Extract movie IDs
+            for movie in data.get('results', []):
+                movie_ids.append(movie['id'])
+                
+            # Rate limiting
+            time.sleep(0.25)  # 4 requests per second max
+            
+        logger.info(f"Collected {len(movie_ids)} movie IDs from {page} pages")
+        return movie_ids
+    
+    def get_movie_details(self, movie_id: int) -> Optional[dict]:
+        """Get detailed movie information"""
+        return self._make_request(f"/movie/{movie_id}")
+    
+    def get_movie_credits(self, movie_id: int) -> Optional[dict]:
+        """Get movie cast and crew"""
+        return self._make_request(f"/movie/{movie_id}/credits")
+
+def fetch_movie_data(tmdb_client: TMDBClient, movie_ids: List[int], max_workers: int = 5) -> Dict[int, dict]:
+    """Fetch detailed data for all movies with controlled parallelization"""
+    movies_data = {}
+    
+    def fetch_single_movie(movie_id: int) -> tuple:
+        """Fetch details and credits for a single movie"""
+        try:
+            # Get basic details
+            details = tmdb_client.get_movie_details(movie_id)
+            if not details:
+                return movie_id, None
+                
+            # Get credits
+            credits = tmdb_client.get_movie_credits(movie_id)
+            if credits:
+                details['credits'] = credits
+                
+            return movie_id, details
+            
+        except Exception as e:
+            logger.error(f"Error fetching movie {movie_id}: {e}")
+            return movie_id, None
+    
+    # Process movies in batches with controlled parallelization
+    batch_size = 50
+    total_movies = len(movie_ids)
+    
+    for i in range(0, total_movies, batch_size):
+        batch = movie_ids[i:i + batch_size]
+        logger.info(f"Processing batch {i//batch_size + 1}/{(total_movies-1)//batch_size + 1} ({len(batch)} movies)")
+        
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {executor.submit(fetch_single_movie, movie_id): movie_id for movie_id in batch}
+            
+            for future in as_completed(futures):
+                movie_id, movie_data = future.result()
+                if movie_data:
+                    movies_data[movie_id] = movie_data
+                    
+        # Sleep between batches to be respectful to API
+        time.sleep(1)
+        
+    logger.info(f"Successfully fetched data for {len(movies_data)}/{total_movies} movies")
+    return movies_data
+
+def create_composite_text(movie_data: Dict) -> str:
+    """Create composite text for embedding from movie data"""
+    parts = []
+    
+    # Title
+    if movie_data.get('title'):
+        parts.append(f"Title: {movie_data['title']}")
+    
+    # Tagline
+    if movie_data.get('tagline'):
+        parts.append(f"Tagline: {movie_data['tagline']}")
+    
+    # Overview
+    if movie_data.get('overview'):
+        parts.append(f"Overview: {movie_data['overview']}")
+    
+    # Release date
+    if movie_data.get('release_date'):
+        parts.append(f"Release Date: {movie_data['release_date']}")
+    
+    # Original language
+    if movie_data.get('original_language'):
+        parts.append(f"Language: {movie_data['original_language']}")
+    
+    # Spoken languages
+    if movie_data.get('spoken_languages'):
+        languages = [lang.get('iso_639_1', '') for lang in movie_data['spoken_languages'] if lang.get('iso_639_1')]
+        if languages:
+            parts.append(f"Spoken Languages: {', '.join(languages)}")
+    
+    # Genres
+    if movie_data.get('genres'):
+        genres = [genre['name'] for genre in movie_data['genres']]
+        parts.append(f"Genres: {', '.join(genres)}")
+    
+    # Production companies
+    if movie_data.get('production_companies'):
+        companies = [company['name'] for company in movie_data['production_companies']]
+        if companies:
+            parts.append(f"Production Companies: {', '.join(companies)}")
+    
+    # Production countries
+    if movie_data.get('production_countries'):
+        countries = [country['name'] for country in movie_data['production_countries']]
+        if countries:
+            parts.append(f"Production Countries: {', '.join(countries)}")
+    
+    # Budget (only if > 0)
+    if movie_data.get('budget') and movie_data['budget'] > 0:
+        parts.append(f"Budget: ${movie_data['budget']:,}")
+    
+    # Popularity
+    if movie_data.get('popularity'):
+        parts.append(f"Popularity: {movie_data['popularity']}")
+    
+    # Vote average
+    if movie_data.get('vote_average'):
+        parts.append(f"Vote Average: {movie_data['vote_average']}")
+    
+    # Vote count
+    if movie_data.get('vote_count'):
+        parts.append(f"Vote Count: {movie_data['vote_count']}")
+    
+    # Director(s)
+    if movie_data.get('credits', {}).get('crew'):
+        directors = [person['name'] for person in movie_data['credits']['crew'] if person['job'] == 'Director']
+        if directors:
+            parts.append(f"Director: {', '.join(directors)}")
+    
+    # Top 5 cast
+    if movie_data.get('credits', {}).get('cast'):
+        top_cast = [person['name'] for person in movie_data['credits']['cast'][:5]]
+        if top_cast:
+            parts.append(f"Cast: {', '.join(top_cast)}")
+    
+    return " / ".join(parts)
+
+def get_embeddings_batch(texts: List[str], client: OpenAI, model: str = "text-embedding-3-small") -> List[List[float]]:
+    """Get embeddings for a batch of texts with retry"""
+    max_retries = 3
+    
+    for attempt in range(max_retries):
+        try:
+            response = client.embeddings.create(
+                input=texts,
+                model=model
+            )
+            return [item.embedding for item in response.data]
+        except Exception as e:
+            logger.error(f"Error getting embeddings (attempt {attempt + 1}): {e}")
+            if attempt < max_retries - 1:
+                time.sleep(2 ** attempt)
+            else:
+                raise
+
+def build_index(max_pages: int = 10, model: str = "text-embedding-3-small", use_faiss: bool = True):
+    """Main function to build the FAISS index and data files"""
+    settings = get_settings()
+    
+    # Initialize clients
+    tmdb_client = TMDBClient(settings.tmdb_api_key)
+    openai_client = OpenAI(api_key=settings.openai_api_key)
+    
+    # Create data directory
+    os.makedirs("app/data", exist_ok=True)
+    
+    # Check for existing movie data checkpoint
+    movies_data = load_checkpoint(MOVIE_DATA_CHECKPOINT)
+    
+    if movies_data is not None:
+        logger.info(f"🔄 Resuming from checkpoint: {len(movies_data)} movies data found")
+    else:
+        # Step 1: Get movie IDs
+        logger.info(f"Fetching movie IDs from TMDB (max {max_pages} pages)...")
+        movie_ids = tmdb_client.get_popular_movies(max_pages=max_pages)
+        
+        if not movie_ids:
+            logger.error("❌ No movie IDs retrieved from TMDB")
+            return
+        
+        # Step 2: Fetch detailed movie data
+        logger.info(f"Fetching detailed data for {len(movie_ids)} movies...")
+        movies_data = fetch_movie_data(tmdb_client, movie_ids)
+        
+        if not movies_data:
+            logger.error("❌ No movie data retrieved")
+            return
+        
+        # Save movie data checkpoint
+        save_checkpoint(movies_data, MOVIE_DATA_CHECKPOINT)
+    
+    # Step 3: Create composite texts and process embeddings in batches
+    logger.info("Creating embeddings...")
+    embeddings = []
+    id_map = {}
+    movie_metadata = {}
+    processed_movie_ids = set()
+    
+    batch_size = 20  # Process 20 texts at a time
+    
+    # Check for existing embedding progress
+    embedding_checkpoint = load_checkpoint(EMBEDDINGS_CHECKPOINT)
+    metadata_checkpoint = load_checkpoint(METADATA_CHECKPOINT)
+    
+    if embedding_checkpoint is not None and metadata_checkpoint is not None:
+        embeddings = embedding_checkpoint['embeddings']
+        id_map = embedding_checkpoint['id_map']
+        processed_movie_ids = set(embedding_checkpoint['processed_movie_ids'])
+        movie_metadata = metadata_checkpoint
+        logger.info(f"🔄 Resuming embeddings from checkpoint: {len(embeddings)} embeddings found")
+    else:
+        logger.info("Starting embeddings from scratch")
+    
+    # Process remaining movies
+    remaining_movies = {k: v for k, v in movies_data.items() if k not in processed_movie_ids}
+    logger.info(f"Processing {len(remaining_movies)} remaining movies")
+    
+    composite_texts = []
+    current_movie_ids = []
+    
+    for movie_id, movie_data in remaining_movies.items():
+        # Create composite text
+        composite_text = create_composite_text(movie_data)
+        composite_texts.append(composite_text)
+        current_movie_ids.append(movie_id)
+        
+        # Store metadata
+        release_year = 0
+        if movie_data.get("release_date"):
+            try:
+                release_year = int(movie_data["release_date"][:4])
+            except (ValueError, IndexError):
+                release_year = 0
+                
+        movie_metadata[str(movie_id)] = {
+            "id": movie_id,
+            "title": movie_data.get("title", ""),
+            "year": release_year,
+            "poster_path": movie_data.get("poster_path"),
+            "release_date": movie_data.get("release_date"),
+            "genres": [g["name"] for g in movie_data.get("genres", [])]
+        }
+        
+        # Process batch when full
+        if len(composite_texts) >= batch_size:
+            logger.info(f"Processing embedding batch ({len(embeddings)} done, {len(composite_texts)} in batch)")
+            
+            try:
+                batch_embeddings = get_embeddings_batch(composite_texts, openai_client, model)
+                embeddings.extend(batch_embeddings)
+                
+                # Update ID mapping and processed set
+                for i, mid in enumerate(current_movie_ids):
+                    id_map[str(mid)] = len(id_map)
+                    processed_movie_ids.add(mid)
+                
+                # Save progress checkpoints
+                embedding_data = {
+                    'embeddings': embeddings,
+                    'id_map': id_map,
+                    'processed_movie_ids': list(processed_movie_ids)
+                }
+                save_checkpoint(embedding_data, EMBEDDINGS_CHECKPOINT)
+                save_checkpoint(movie_metadata, METADATA_CHECKPOINT)
+                
+                # Clear batch
+                composite_texts = []
+                current_movie_ids = []
+                
+                # Sleep between batches
+                time.sleep(0.5)
+                
+            except Exception as e:
+                logger.error(f"Failed to process batch: {e}")
+                logger.info("Progress has been saved, you can restart the script to resume")
+                return
+    
+    # Process remaining texts
+    if composite_texts:
+        logger.info(f"Processing final embedding batch ({len(composite_texts)} texts)")
+        try:
+            batch_embeddings = get_embeddings_batch(composite_texts, openai_client, model)
+            embeddings.extend(batch_embeddings)
+            
+            for i, mid in enumerate(current_movie_ids):
+                id_map[str(mid)] = len(id_map)
+                processed_movie_ids.add(mid)
+            
+            # Save final progress
+            embedding_data = {
+                'embeddings': embeddings,
+                'id_map': id_map,
+                'processed_movie_ids': list(processed_movie_ids)
+            }
+            save_checkpoint(embedding_data, EMBEDDINGS_CHECKPOINT)
+            save_checkpoint(movie_metadata, METADATA_CHECKPOINT)
+            
+        except Exception as e:
+            logger.error(f"Failed to process final batch: {e}")
+            logger.info("Progress has been saved, you can restart the script to resume")
+            return
+    
+    if not embeddings:
+        logger.error("❌ No embeddings generated")
+        return
+    
+    logger.info(f"Generated {len(embeddings)} embeddings")
+    
+    # Step 4: Save embeddings as numpy array
+    embeddings_array = np.array(embeddings, dtype=np.float32)
+    np.save("app/data/movies.npy", embeddings_array)
+    logger.info(f"Saved embeddings matrix: {embeddings_array.shape}")
+    
+    # Step 5: Build and save FAISS index
+    if use_faiss:
+        logger.info("Building FAISS index...")
+        dimension = embeddings_array.shape[1]
+        
+        # Choose index type based on size
+        if len(embeddings) < 10000:
+            # For smaller datasets, use flat index
+            index = faiss.IndexFlatL2(dimension)
+        else:
+            # For larger datasets, use IVF index
+            nlist = min(int(np.sqrt(len(embeddings))), 1000)
+            quantizer = faiss.IndexFlatL2(dimension)
+            index = faiss.IndexIVFFlat(quantizer, dimension, nlist)
+            # Train the index
+            index.train(embeddings_array)
+        
+        index.add(embeddings_array)
+        faiss.write_index(index, "app/data/faiss.index")
+        logger.info(f"FAISS index saved (type: {type(index).__name__}, dimension: {dimension})")
+    
+    # Step 6: Save metadata files
+    with open("app/data/id_map.json", "w") as f:
+        json.dump(id_map, f)
+    
+    with open("app/data/movie_metadata.json", "w") as f:
+        json.dump(movie_metadata, f)
+    
+    logger.info("✅ Index built successfully!")
+    logger.info(f"   - {len(embeddings)} movies indexed")
+    logger.info(f"   - Embedding model: {model}")
+    logger.info(f"   - Files saved in app/data/")
+    logger.info(f"     * movies.npy: embeddings matrix")
+    logger.info(f"     * id_map.json: TMDB ID to matrix position mapping")
+    logger.info(f"     * movie_metadata.json: movie metadata")
+    if use_faiss:
+        logger.info(f"     * faiss.index: FAISS search index")
+    
+    # Cleanup checkpoints
+    cleanup_checkpoints()
+
+# Remove the old functions that are no longer needed
+# create_movie_embedding and load_movie_data are replaced by the new implementation
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Build movie embeddings index from TMDB data")
+    parser.add_argument("--max-pages", type=int, default=10, 
+                       help="Maximum pages to fetch from TMDB popular movies (default: 10)")
+    parser.add_argument("--model", type=str, default="text-embedding-3-small",
+                       help="OpenAI embedding model to use (default: text-embedding-3-small)")
+    parser.add_argument("--no-faiss", action="store_true",
+                       help="Skip building FAISS index")
+    
+    args = parser.parse_args()
+    
+    build_index(
+        max_pages=args.max_pages,
+        model=args.model,
+        use_faiss=not args.no_faiss
+    )

app/main.py

@@ -0,0 +1,303 @@
+import os
+import json
+import numpy as np
+import faiss
+from fastapi import FastAPI, HTTPException, Depends, status
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import BaseModel
+from typing import List, Optional
+import logging
+import time
+
+# Configure logging
+logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO").upper())
+logger = logging.getLogger(__name__)
+
+# Security
+security = HTTPBearer()
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    expected_token = os.getenv("API_TOKEN")
+    if not expected_token:
+        raise HTTPException(status_code=500, detail="API token not configured")
+    if credentials.credentials != expected_token:
+        raise HTTPException(status_code=401, detail="Invalid token")
+    return credentials.credentials
+
+# Pydantic models
+class ExploreRequest(BaseModel):
+    liked_ids: List[int]
+    disliked_ids: List[int] = []
+    top_k: int = 400
+
+class MovieResult(BaseModel):
+    id: int
+    title: str
+    year: int
+    poster_path: Optional[str]
+    genres: List[str]
+    coords: List[float]
+
+class ExploreResponse(BaseModel):
+    movies: List[MovieResult]
+    bary: List[float]
+    center: List[float]
+
+# Global variables for loaded data
+vectors = None
+id_map = None
+faiss_index = None
+movie_metadata = None
+
+def load_data():
+    """Load FAISS index, vectors, and metadata on startup"""
+    global vectors, id_map, faiss_index, movie_metadata
+    
+    try:
+        # Load vectors
+        vectors = np.load("app/data/movies.npy")
+        logger.info(f"Loaded {vectors.shape[0]} movie vectors of dimension {vectors.shape[1]}")
+        
+        # Load ID mapping
+        with open("app/data/id_map.json", "r") as f:
+            id_map = json.load(f)
+        logger.info(f"Loaded ID mapping for {len(id_map)} movies")
+        
+        # Load FAISS index
+        faiss_index = faiss.read_index("app/data/faiss.index")
+        logger.info(f"Loaded FAISS index with {faiss_index.ntotal} vectors")
+        
+        # Load movie metadata
+        with open("app/data/movie_metadata.json", "r") as f:
+            movie_metadata = json.load(f)
+        logger.info(f"Loaded metadata for {len(movie_metadata)} movies")
+        
+    except Exception as e:
+        logger.error(f"Failed to load data: {e}")
+        raise
+
+def build_plane(likes: np.ndarray, dislikes: np.ndarray = None, dim: int = 2):
+    """
+    Build user subspace from liked/disliked movies
+    Returns (axes, center) where axes is 2xD orthonormal matrix
+    """
+    n_likes = likes.shape[0] if likes is not None else 0
+    d = vectors.shape[1]
+    
+    # Compute composite vector: +liked - 0.5*disliked
+    if n_likes == 0:
+        # Cold start: use global average
+        center = vectors.mean(0)
+        # Create random orthonormal basis
+        axes = np.random.randn(dim, d)
+        axes[0] /= np.linalg.norm(axes[0])
+        for i in range(1, dim):
+            for j in range(i):
+                axes[i] -= np.dot(axes[i], axes[j]) * axes[j]
+            axes[i] /= np.linalg.norm(axes[i])
+    else:
+        # Compute composite from likes and dislikes
+        composite = likes.mean(0)
+        if dislikes is not None and dislikes.shape[0] > 0:
+            composite -= 0.5 * dislikes.mean(0)
+        
+        if n_likes == 1:
+            # One like: use as center, random orthogonal axes
+            center = composite
+            axis1 = np.random.randn(d)
+            axis1 /= np.linalg.norm(axis1)
+            axis2 = np.random.randn(d)
+            axis2 -= np.dot(axis2, axis1) * axis1
+            axis2 /= np.linalg.norm(axis2)
+            axes = np.vstack([axis1, axis2])
+        elif n_likes == 2:
+            # Two likes: line between them
+            center = likes.mean(0)
+            axis1 = likes[1] - likes[0]
+            axis1 /= np.linalg.norm(axis1)
+            axis2 = np.random.randn(d)
+            axis2 -= np.dot(axis2, axis1) * axis1
+            axis2 /= np.linalg.norm(axis2)
+            axes = np.vstack([axis1, axis2])
+        else:
+            # 3+ likes: PCA plane
+            center = likes.mean(0)
+            likes_centered = likes - center
+            u, s, vt = np.linalg.svd(likes_centered, full_matrices=False)
+            axes = vt[:2]  # First 2 principal components
+    
+    return axes, center
+
+def assign_spiral_coords(n_movies: int):
+    """
+    Assign 2D grid coordinates in outward spiral pattern
+    Returns array of shape (n_movies, 2) with integer coordinates
+    """
+    coords = np.zeros((n_movies, 2), dtype=int)
+    if n_movies == 0:
+        return coords
+    
+    coords[0] = [0, 0]  # Start at origin
+    
+    if n_movies == 1:
+        return coords
+    
+    # Spiral pattern: right, up, left, down, repeat with increasing distances
+    dx, dy = [1, 0, -1, 0], [0, 1, 0, -1]
+    direction = 0
+    steps = 1
+    x, y = 0, 0
+    idx = 1
+    
+    while idx < n_movies:
+        for _ in range(2):  # Each step count is used twice (except the first)
+            for _ in range(steps):
+                if idx >= n_movies:
+                    break
+                x += dx[direction]
+                y += dy[direction]
+                coords[idx] = [x, y]
+                idx += 1
+            direction = (direction + 1) % 4
+            if idx >= n_movies:
+                break
+        steps += 1
+    
+    return coords
+
+def compute_barycenter(liked_indices: List[int], coords: np.ndarray):
+    """Compute barycenter of liked movies in 2D grid"""
+    if not liked_indices:
+        return [0.0, 0.0]
+    
+    liked_coords = coords[liked_indices]
+    bary = liked_coords.mean(0)
+    return bary.tolist()
+
+# FastAPI app setup
+app = FastAPI(title="Karl-Movie Vector Backend", version="1.0.0")
+
+# CORS configuration
+DEV_ORIGINS = [
+    "http://localhost:5173",
+    "http://127.0.0.1:5173",
+    "http://localhost:8888",
+    "https://*.bolt.run",
+    "https://*.stackblitz.io",
+]
+
+PROD_ORIGINS = ["https://karl.movie"]
+
+origins = DEV_ORIGINS if os.getenv("ENV") != "prod" else PROD_ORIGINS
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_methods=["POST", "GET"],
+    allow_headers=["*"],
+)
+
+@app.on_event("startup")
+async def startup_event():
+    """Load data on startup"""
+    load_data()
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {"status": "healthy", "vectors_loaded": vectors is not None}
+
+@app.post("/explore", response_model=ExploreResponse)
+async def explore(
+    request: ExploreRequest,
+    token: str = Depends(verify_token)
+):
+    """
+    Main endpoint: find movies closest to user's preference subspace
+    """
+    start_time = time.time()
+    
+    try:
+        # Convert TMDB IDs to internal indices
+        liked_indices = []
+        disliked_indices = []
+        
+        for tmdb_id in request.liked_ids:
+            if str(tmdb_id) in id_map:
+                liked_indices.append(id_map[str(tmdb_id)])
+            else:
+                logger.warning(f"TMDB ID {tmdb_id} not found in index")
+        
+        for tmdb_id in request.disliked_ids:
+            if str(tmdb_id) in id_map:
+                disliked_indices.append(id_map[str(tmdb_id)])
+            else:
+                logger.warning(f"TMDB ID {tmdb_id} not found in index")
+        
+        # Get embedding vectors
+        liked_vectors = vectors[liked_indices] if liked_indices else None
+        disliked_vectors = vectors[disliked_indices] if disliked_indices else None
+        
+        # Build user subspace
+        axes, center = build_plane(liked_vectors, disliked_vectors)
+        
+        # Project all vectors onto the 2D subspace
+        projections = np.dot(vectors - center, axes.T)  # Shape: (N, 2)
+        
+        # Reconstruct vectors in original space
+        reconstructed = np.dot(projections, axes) + center
+        
+        # Compute distances to subspace (residuals)
+        residuals = np.linalg.norm(vectors - reconstructed, axis=1)
+        
+        # Get top-k closest movies
+        top_k_indices = np.argpartition(residuals, min(request.top_k, len(residuals)))[:request.top_k]
+        top_k_indices = top_k_indices[np.argsort(residuals[top_k_indices])]
+        
+        # Assign spiral coordinates
+        spiral_coords = assign_spiral_coords(len(top_k_indices))
+        
+        # Compute barycenter of liked movies
+        liked_positions = [i for i, idx in enumerate(top_k_indices) if idx in liked_indices]
+        bary = compute_barycenter(liked_positions, spiral_coords)
+        
+        # Translate grid so barycenter is at origin
+        spiral_coords = spiral_coords - np.array(bary)
+        
+        # Build response
+        movies = []
+        reverse_id_map = {v: k for k, v in id_map.items()}
+        
+        for i, movie_idx in enumerate(top_k_indices):
+            tmdb_id = int(reverse_id_map[movie_idx])
+            metadata = movie_metadata.get(str(tmdb_id), {})
+            
+            movie = MovieResult(
+                id=tmdb_id,
+                title=metadata.get("title", f"Movie {tmdb_id}"),
+                year=metadata.get("year", 0),
+                poster_path=metadata.get("poster_path"),
+                genres=metadata.get("genres", []),
+                coords=spiral_coords[i].tolist()
+            )
+            movies.append(movie)
+        
+        response = ExploreResponse(
+            movies=movies,
+            bary=[0.0, 0.0],  # Always [0,0] since we translated
+            center=center.tolist()
+        )
+        
+        elapsed = time.time() - start_time
+        logger.info(f"Explore request processed in {elapsed:.3f}s - {len(request.liked_ids)} likes, {len(request.disliked_ids)} dislikes, {len(movies)} results")
+        
+        return response
+        
+    except Exception as e:
+        logger.error(f"Error processing explore request: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app/settings.py

@@ -0,0 +1,35 @@
+"""
+Settings and environment configuration
+"""
+import os
+from functools import lru_cache
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    """Application settings"""
+    
+    # OpenAI API key for embeddings
+    openai_api_key: str
+    
+    # TMDB API key for movie data
+    tmdb_api_key: str
+    
+    # API authentication token
+    api_token: str
+    
+    # Environment (dev/prod)
+    env: str = "dev"
+    
+    # Logging level
+    log_level: str = "INFO"
+    
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+
+
+@lru_cache()
+def get_settings() -> Settings:
+    """Get cached settings instance"""
+    return Settings()

app/test_api.py

@@ -0,0 +1,80 @@
+"""
+Test l'API /explore avec des exemples de films
+"""
+import requests
+import json
+
+# Configuration
+API_URL = "http://localhost:8000"
+API_TOKEN = "your-api-token"  # Remplacez par votre token
+
+def test_explore_endpoint():
+    """Test l'endpoint /explore avec différents scénarios"""
+    
+    # Lire les métadonnées pour avoir des IDs de test
+    with open("app/data/movie_metadata.json", "r") as f:
+        metadata = json.load(f)
+    
+    # Prendre les premiers films comme exemples
+    movie_ids = list(metadata.keys())[:5]
+    print(f"Films de test disponibles : {[metadata[mid]['title'] for mid in movie_ids]}")
+    
+    # Test 1: Recherche avec 1 film aimé
+    print("\n🎬 Test 1: Recherche avec 1 film aimé")
+    test_request = {
+        "liked_ids": [int(movie_ids[0])],
+        "disliked_ids": [],
+        "top_k": 10
+    }
+    
+    try:
+        response = requests.post(
+            f"{API_URL}/explore",
+            json=test_request,
+            headers={"Authorization": f"Bearer {API_TOKEN}"}
+        )
+        
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Trouvé {len(data['movies'])} films similaires")
+            print(f"Film aimé: {metadata[movie_ids[0]]['title']}")
+            print("Films recommandés:")
+            for movie in data['movies'][:3]:
+                print(f"  - {movie['title']} ({movie['year']}) - {movie['genres']}")
+        else:
+            print(f"❌ Erreur {response.status_code}: {response.text}")
+            
+    except Exception as e:
+        print(f"❌ Erreur de connexion: {e}")
+        print("💡 Vérifiez que votre API_TOKEN est correct dans le .env")
+
+    # Test 2: Recherche avec 2 films aimés
+    print("\n🎬 Test 2: Recherche avec 2 films aimés")
+    test_request = {
+        "liked_ids": [int(movie_ids[0]), int(movie_ids[1])],
+        "disliked_ids": [],
+        "top_k": 10
+    }
+    
+    try:
+        response = requests.post(
+            f"{API_URL}/explore",
+            json=test_request,
+            headers={"Authorization": f"Bearer {API_TOKEN}"}
+        )
+        
+        if response.status_code == 200:
+            data = response.json()
+            print(f"✅ Trouvé {len(data['movies'])} films similaires")
+            print(f"Films aimés: {metadata[movie_ids[0]]['title']}, {metadata[movie_ids[1]]['title']}")
+            print("Barycenter:", data['bary'])
+        else:
+            print(f"❌ Erreur {response.status_code}: {response.text}")
+            
+    except Exception as e:
+        print(f"❌ Erreur: {e}")
+
+if __name__ == "__main__":
+    print("🧪 Test de l'API /explore")
+    print("=" * 40)
+    test_explore_endpoint()

app/test_setup.py

@@ -0,0 +1,121 @@
+"""
+Test script for TMDB data loading and embedding generation
+Run this to validate your setup before building the full index
+"""
+import os
+import sys
+import json
+from settings import get_settings
+from build_index import TMDBClient, create_composite_text, get_embeddings_batch
+from openai import OpenAI
+
+def test_tmdb_connection():
+    """Test TMDB API connection"""
+    print("🔍 Testing TMDB API connection...")
+    
+    try:
+        settings = get_settings()
+        tmdb_client = TMDBClient(settings.tmdb_api_key)
+        
+        # Test getting popular movies (just first page)
+        movie_ids = tmdb_client.get_popular_movies(max_pages=1)
+        
+        if movie_ids:
+            print(f"✅ Successfully fetched {len(movie_ids)} movie IDs from TMDB")
+            
+            # Test getting details for first movie
+            movie_data = tmdb_client.get_movie_details(movie_ids[0])
+            if movie_data:
+                print(f"✅ Successfully fetched details for movie: {movie_data.get('title', 'Unknown')}")
+                
+                # Test getting credits
+                credits = tmdb_client.get_movie_credits(movie_ids[0])
+                if credits:
+                    print(f"✅ Successfully fetched credits (cast: {len(credits.get('cast', []))}, crew: {len(credits.get('crew', []))})")
+                else:
+                    print("⚠️  Could not fetch credits")
+                    
+                return movie_data, credits
+            else:
+                print("❌ Could not fetch movie details")
+        else:
+            print("❌ Could not fetch movie IDs")
+            
+    except Exception as e:
+        print(f"❌ TMDB API error: {e}")
+        
+    return None, None
+
+def test_composite_text(movie_data, credits):
+    """Test composite text creation"""
+    print("\n📝 Testing composite text creation...")
+    
+    if movie_data:
+        # Add credits to movie data
+        if credits:
+            movie_data['credits'] = credits
+            
+        composite_text = create_composite_text(movie_data)
+        print(f"✅ Generated composite text ({len(composite_text)} chars)")
+        print(f"Preview: {composite_text[:200]}...")
+        return composite_text
+    else:
+        print("❌ No movie data to test")
+        return None
+
+def test_embeddings(composite_text):
+    """Test embedding generation"""
+    print("\n🤖 Testing embedding generation...")
+    
+    if composite_text:
+        try:
+            settings = get_settings()
+            openai_client = OpenAI(api_key=settings.openai_api_key)
+            
+            embeddings = get_embeddings_batch([composite_text], openai_client)
+            
+            if embeddings:
+                embedding = embeddings[0]
+                print(f"✅ Generated embedding (dimension: {len(embedding)})")
+                print(f"Sample values: {embedding[:5]}...")
+                return embedding
+            else:
+                print("❌ No embeddings generated")
+                
+        except Exception as e:
+            print(f"❌ Embedding error: {e}")
+    else:
+        print("❌ No composite text to test")
+        
+    return None
+
+def main():
+    """Run all tests"""
+    print("🎬 Karl Movie Vector Backend - Test Suite")
+    print("=" * 50)
+    
+    # Test environment variables
+    print("🔧 Checking environment variables...")
+    try:
+        settings = get_settings()
+        print(f"✅ OpenAI API key: {'sk-...' + settings.openai_api_key[-10:] if settings.openai_api_key else 'Not set'}")
+        print(f"✅ TMDB API key: {'...' + settings.tmdb_api_key[-10:] if settings.tmdb_api_key else 'Not set'}")
+    except Exception as e:
+        print(f"❌ Settings error: {e}")
+        print("Make sure you have a .env file with OPENAI_API_KEY and TMDB_API_KEY")
+        return
+    
+    # Run tests
+    movie_data, credits = test_tmdb_connection()
+    composite_text = test_composite_text(movie_data, credits)
+    embedding = test_embeddings(composite_text)
+    
+    print("\n" + "=" * 50)
+    if movie_data and composite_text and embedding:
+        print("🎉 All tests passed! You can now run the full build:")
+        print("   python app/build_index.py --max-pages 3")
+    else:
+        print("❌ Some tests failed. Check your API keys and internet connection.")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,12 @@
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+numpy==1.24.4
+faiss-cpu==1.7.4
+openai==1.51.0
+pydantic==2.11.5
+pydantic-settings==2.9.1
+python-multipart==0.0.6
+requests==2.31.0
+scikit-learn==1.3.2
+python-dotenv==1.0.0
+httpx==0.27.0