vector_store.py

# vector_store.py - Vector store integration with Pinecone
"""
Vector store integration for legal document embeddings using InLegalBERT and Pinecone
"""
import os
import numpy as np
from typing import List, Dict, Any
from langchain_pinecone import PineconeVectorStore
from langchain.embeddings.base import Embeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter


class InLegalBERTEmbeddings(Embeddings):
    """Custom LangChain embeddings wrapper for InLegalBERT"""
    
    def __init__(self, model):
        self.model = model
    
    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed a list of documents"""
        return self.model.encode(texts).tolist()
    
    def embed_query(self, text: str) -> List[float]:
        """Embed a single query"""
        return self.model.encode([text])[0].tolist()


class LegalDocumentVectorStore:
    """Manages vector storage for legal documents"""
    
    def __init__(self):
        self.index_name = 'legal-documents'
        self.dimension = 768  # InLegalBERT dimension
        self._initialized = False
        self.clause_tagger = None
        self.pc = None
    
    def _initialize_pinecone(self):
        """Initialize Pinecone connection"""
        if self._initialized:
            return
            
        PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
        
        if not PINECONE_API_KEY:
            raise ValueError("PINECONE_API_KEY environment variable not set")
        
        # Use modern Pinecone API
        from pinecone import Pinecone, ServerlessSpec
        self.pc = Pinecone(api_key=PINECONE_API_KEY)
        
        # Create index if doesn't exist
        existing_indexes = [index_info["name"] for index_info in self.pc.list_indexes()]
        if self.index_name not in existing_indexes:
            self.pc.create_index(
                name=self.index_name,
                dimension=self.dimension,
                metric='cosine',
                spec=ServerlessSpec(cloud='aws', region='us-east-1')
            )
            print(f"✅ Created Pinecone index: {self.index_name}")
        
        self._initialized = True

    def _normalize_embedding(self, embedding):
        """Ensure embedding is always a list of floats"""
        if embedding is None:
            return None
        if isinstance(embedding, np.ndarray):
            return embedding.tolist()
        if isinstance(embedding, list):
            # Already a Python list
            return embedding
        # Fallback: try converting if it's a torch tensor or similar
        try:
            return embedding.tolist()
        except Exception:
            return list(embedding)

    def save_document_embeddings_optimized(
        self,
        chunk_data: List[Dict],
        document_id: str,
        analysis_results: Dict[str, Any]
    ) -> bool:
        """Save embeddings using pre-computed vectors with proper text storage"""
        try:
            self._initialize_pinecone()

            # Normalize embeddings safely
            valid_chunks = [c for c in chunk_data if c.get("embedding") is not None]
            if not valid_chunks:
                print("⚠️ No embeddings found in chunk_data")
                return False

            vectors = []
            for i, chunk_info in enumerate(valid_chunks):
                normalized_embedding = self._normalize_embedding(chunk_info["embedding"])
                if normalized_embedding is None:
                    continue

                metadata = {
                    'document_id': document_id,
                    'chunk_index': i,
                    'total_chunks': len(chunk_data),
                    'source': 'legal_document',
                    'has_key_clauses': len(analysis_results.get('key_clauses', [])) > 0,
                    'risk_count': len(analysis_results.get('risky_terms', [])),
                    'embedding_model': 'InLegalBERT',
                    'timestamp': str(np.datetime64('now')),
                    'text': chunk_info["text"]  # Store text in metadata for retrieval
                }

                vectors.append({
                    "id": f"{document_id}_chunk_{i}",
                    "values": normalized_embedding,
                    "metadata": metadata
                })

            # Add to Pinecone
            index = self.pc.Index(self.index_name)
            index.upsert(vectors=vectors)

            print(f"✅ Saved {len(vectors)} pre-computed embeddings with text to Pinecone")
            return True

        except Exception as e:
            print(f"❌ Error saving pre-computed embeddings: {e}")
            return False

    def get_retriever(self, clause_tagger, document_id: str = None):
        """Get retriever for chat functionality with improved settings"""
        try:
            self._initialize_pinecone()

            legal_embeddings = InLegalBERTEmbeddings(clause_tagger.embedding_model)
            index = self.pc.Index(self.index_name)

            vectorstore = PineconeVectorStore(
                index=index,
                embedding=legal_embeddings,
                text_key="text"  # Use text stored in metadata
            )

            # Configure search parameters
            search_kwargs = {'k': 10}
            if document_id:
                search_kwargs['filter'] = {'document_id': document_id}

            return vectorstore.as_retriever(
                search_type="similarity",
                search_kwargs=search_kwargs
            )

        except Exception as e:
            print(f"❌ Error creating retriever: {e}")
            return None


# Global instance
vector_store = LegalDocumentVectorStore()